Skip to content

Apache DataFusion SQL Query Engine

fix(ffi): forward distinct_handling through FFI_AggregateUDF (#25876)

Bharadwaj Pendyalaac279ebf5cee
diff --git a/datafusion/ffi/integration-tests/tests/cases/udaf.rs b/datafusion/ffi/integration-tests/tests/cases/udaf.rs
index 71aca901d0..7805f8fcc9 100644
--- a/datafusion/ffi/integration-tests/tests/cases/udaf.rs
+++ b/datafusion/ffi/integration-tests/tests/cases/udaf.rs
@@ -20,10 +20,10 @@ mod tests {
use arrow::array::{Float64Array, record_batch};
use datafusion::error::Result;
- use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl};
+ use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl, DistinctHandling};
use datafusion::prelude::{SessionContext, col};
use datafusion_catalog::MemTable;
- use datafusion_expr::{ScalarUDF, ScalarUDFImpl};
+ use datafusion_expr::{ExprFunctionExt, ScalarUDF, ScalarUDFImpl};
use datafusion_ffi::tests::utils::get_module;
#[tokio::test]
@@ -79,6 +79,71 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_distinct_handling() -> Result<()> {
+ let module = get_module()?;
+
+ let ffi_stddev_func = (module.create_stddev_udaf)();
+ let foreign_stddev_func: Arc<dyn AggregateUDFImpl> = (&ffi_stddev_func).into();
+ assert_eq!(
+ foreign_stddev_func.distinct_handling(),
+ DistinctHandling::Unsupported
+ );
+
+ let ffi_sum_func = (module.create_sum_udaf)();
+ let foreign_sum_func: Arc<dyn AggregateUDFImpl> = (&ffi_sum_func).into();
+ assert_eq!(
+ foreign_sum_func.distinct_handling(),
+ DistinctHandling::Sensitive
+ );
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_foreign_min_distinct_drops_distinct() -> Result<()> {
+ let module = get_module()?;
+
+ let ffi_min_func = (module.create_min_udaf)();
+ let foreign_min_func: Arc<dyn AggregateUDFImpl> = (&ffi_min_func).into();
+
+ let udaf = AggregateUDF::new_from_shared_impl(foreign_min_func);
+
+ let ctx = SessionContext::default();
+ let record_batch = record_batch!(
+ ("a", Int32, vec![1, 2, 2, 4, 4, 4, 4]),
+ ("b", Float64, vec![3.0, 2.0, 2.0, 4.0, 1.0, 1.0, 4.0])
+ )
+ .unwrap();
+
+ let df = ctx
+ .read_batch(record_batch)?
+ .aggregate(
+ vec![col("a")],
+ vec![udaf.call(vec![col("b")]).distinct().build()?.alias("min_b")],
+ )?
+ .sort_by(vec![col("a")])?;
+
+ let plan = df.clone().into_optimized_plan()?;
+ assert_eq!(
+ plan.display_indent().to_string(),
+ "Sort: ?table?.a ASC NULLS LAST\
+ \n Aggregate: groupBy=[[?table?.a]], aggr=[[min(?table?.b) AS min_b]]\
+ \n TableScan: ?table? projection=[a, b]"
+ );
+
+ let result = df.collect().await?;
+
+ let expected = record_batch!(
+ ("a", Int32, vec![1, 2, 4]),
+ ("min_b", Float64, vec![3.0, 2.0, 1.0])
+ )?;
+
+ assert_eq!(result[0], expected);
+
+ Ok(())
+ }
+
#[tokio::test]
async fn test_ffi_grouping_udaf() -> Result<()> {
let module = get_module()?;
diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs
index a334bd6d21..99cc611acc 100644
--- a/datafusion/ffi/src/tests/mod.rs
+++ b/datafusion/ffi/src/tests/mod.rs
@@ -36,9 +36,9 @@ use datafusion_physical_expr::{
use datafusion_physical_plan::{ExecutionPlan, Partitioning};
use sync_provider::create_sync_table_provider;
use udf_udaf_udwf::{
- create_ffi_abs_func, create_ffi_first_value_func, create_ffi_random_func,
- create_ffi_rank_func, create_ffi_stddev_func, create_ffi_sum_func,
- create_ffi_table_func,
+ create_ffi_abs_func, create_ffi_first_value_func, create_ffi_min_func,
+ create_ffi_random_func, create_ffi_rank_func, create_ffi_stddev_func,
+ create_ffi_sum_func, create_ffi_table_func,
};
use crate::catalog_provider::FFI_CatalogProvider;
@@ -142,6 +142,9 @@ pub struct ForeignLibraryModule {
/// Create an aggregate UDAF using first_value
pub create_first_value_udaf: extern "C" fn() -> FFI_AggregateUDF,
+
+ /// Create an aggregate UDAF using min
+ pub create_min_udaf: extern "C" fn() -> FFI_AggregateUDF,
}
pub fn create_test_schema() -> Arc<Schema> {
@@ -429,5 +432,6 @@ pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {
create_query_planner: query_planner::create_query_planner,
version: super::version,
create_first_value_udaf: create_ffi_first_value_func,
+ create_min_udaf: create_ffi_min_func,
}
}
diff --git a/datafusion/ffi/src/tests/udf_udaf_udwf.rs b/datafusion/ffi/src/tests/udf_udaf_udwf.rs
index ea7c03b5d6..8603a85c8a 100644
--- a/datafusion/ffi/src/tests/udf_udaf_udwf.rs
+++ b/datafusion/ffi/src/tests/udf_udaf_udwf.rs
@@ -29,6 +29,7 @@ use datafusion_expr::{
use datafusion_functions::math::abs::AbsFunc;
use datafusion_functions::math::random::RandomFunc;
use datafusion_functions_aggregate::first_last::FirstValue;
+use datafusion_functions_aggregate::min_max::Min;
use datafusion_functions_aggregate::stddev::Stddev;
use datafusion_functions_aggregate::sum::Sum;
use datafusion_functions_table::generate_series::RangeFunc;
@@ -210,6 +211,12 @@ pub(crate) extern "C" fn create_ffi_stddev_func() -> FFI_AggregateUDF {
udaf.into()
}
+pub(crate) extern "C" fn create_ffi_min_func() -> FFI_AggregateUDF {
+ let udaf: Arc<AggregateUDF> = Arc::new(Min::new().into());
+
+ udaf.into()
+}
+
pub(crate) extern "C" fn create_ffi_rank_func() -> FFI_WindowUDF {
let udwf: Arc<WindowUDF> = Arc::new(
Rank::new(
diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs
index a103b4af20..df27ec78ba 100644
--- a/datafusion/ffi/src/udaf/mod.rs
+++ b/datafusion/ffi/src/udaf/mod.rs
@@ -24,7 +24,8 @@ use datafusion_common::{DataFusionError, Result, ffi_datafusion_err};
use datafusion_expr::function::AggregateFunctionSimplification;
use datafusion_expr::type_coercion::functions::fields_with_udf;
use datafusion_expr::{
- Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature,
+ Accumulator, AggregateUDF, AggregateUDFImpl, DistinctHandling, GroupsAccumulator,
+ Signature,
};
use datafusion_functions_aggregate_common::accumulator::{
AccumulatorArgs, StateFieldsArgs,
@@ -149,6 +150,10 @@ pub struct FFI_AggregateUDF {
/// FFI equivalent to [`AggregateUDF::supports_null_handling_clause`]
pub supports_null_handling_clause:
unsafe extern "C" fn(udaf: &FFI_AggregateUDF) -> bool,
+
+ /// FFI equivalent to [`AggregateUDF::distinct_handling`]
+ pub distinct_handling:
+ unsafe extern "C" fn(udaf: &FFI_AggregateUDF) -> FFI_DistinctHandling,
}
unsafe impl Send for FFI_AggregateUDF {}
@@ -337,6 +342,12 @@ unsafe extern "C" fn supports_null_handling_clause_fn_wrapper(
unsafe { udaf.inner().supports_null_handling_clause() }
}
+unsafe extern "C" fn distinct_handling_fn_wrapper(
+ udaf: &FFI_AggregateUDF,
+) -> FFI_DistinctHandling {
+ unsafe { udaf.inner().distinct_handling().into() }
+}
+
unsafe extern "C" fn coerce_types_fn_wrapper(
udaf: &FFI_AggregateUDF,
arg_types: SVec<WrappedSchema>,
@@ -412,6 +423,7 @@ impl From<Arc<AggregateUDF>> for FFI_AggregateUDF {
private_data: Box::into_raw(private_data).cast::<c_void>(),
library_marker_id: crate::get_library_marker_id,
supports_null_handling_clause: supports_null_handling_clause_fn_wrapper,
+ distinct_handling: distinct_handling_fn_wrapper,
}
}
}
@@ -610,6 +622,10 @@ impl AggregateUDFImpl for ForeignAggregateUDF {
unsafe { (self.udaf.supports_null_handling_clause)(&self.udaf) }
}
+ fn distinct_handling(&self) -> DistinctHandling {
+ unsafe { (self.udaf.distinct_handling)(&self.udaf).into() }
+ }
+
fn simplify(&self) -> Option<AggregateFunctionSimplification> {
None
}
@@ -655,6 +671,37 @@ impl From<AggregateOrderSensitivity> for FFI_AggregateOrderSensitivity {
}
}
+#[repr(C)]
+#[derive(Debug)]
+pub enum FFI_DistinctHandling {
+ Insensitive,
+ Sensitive,
+ Unsupported,
+}
+
+impl From<FFI_DistinctHandling> for DistinctHandling {
+ fn from(value: FFI_DistinctHandling) -> Self {
+ match value {
+ FFI_DistinctHandling::Insensitive => Self::Insensitive,
+ FFI_DistinctHandling::Sensitive => Self::Sensitive,
+ FFI_DistinctHandling::Unsupported => Self::Unsupported,
+ }
+ }
+}
+
+impl From<DistinctHandling> for FFI_DistinctHandling {
+ fn from(value: DistinctHandling) -> Self {
+ match value {
+ DistinctHandling::Insensitive => Self::Insensitive,
+ DistinctHandling::Sensitive => Self::Sensitive,
+ DistinctHandling::Unsupported => Self::Unsupported,
+ // `DistinctHandling` is non_exhaustive, so map any variant added
+ // later to the default.
+ _ => Self::Sensitive,
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
@@ -798,6 +845,24 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_distinct_handling() -> Result<()> {
+ let min = create_test_foreign_udaf(
+ datafusion::functions_aggregate::min_max::Min::new(),
+ )?;
+ assert_eq!(min.distinct_handling(), DistinctHandling::Insensitive);
+
+ let sum = create_test_foreign_udaf(Sum::new())?;
+ assert_eq!(sum.distinct_handling(), DistinctHandling::Sensitive);
+
+ let stddev = create_test_foreign_udaf(
+ datafusion::functions_aggregate::stddev::Stddev::new(),
+ )?;
+ assert_eq!(stddev.distinct_handling(), DistinctHandling::Unsupported);
+
+ Ok(())
+ }
+
#[test]
fn test_beneficial_ordering() -> Result<()> {
let foreign_udaf = create_test_foreign_udaf(
@@ -870,6 +935,18 @@ mod tests {
test_round_trip_order_sensitivity(AggregateOrderSensitivity::Beneficial);
}
+ #[test]
+ fn test_round_trip_all_distinct_handlings() {
+ for handling in [
+ DistinctHandling::Insensitive,
+ DistinctHandling::Sensitive,
+ DistinctHandling::Unsupported,
+ ] {
+ let ffi_handling: FFI_DistinctHandling = handling.into();
+ assert_eq!(DistinctHandling::from(ffi_handling), handling);
+ }
+ }
+
#[test]
fn test_ffi_udaf_local_bypass() -> Result<()> {
let original_udaf = Sum::new();