Skip to content

Apache DataFusion SQL Query Engine

FFI: plumb `placement` for `FFI_ScalarUDF` (#22608)

Amogh Ramesha7280b87f541
diff --git a/datafusion/ffi/src/lib.rs b/datafusion/ffi/src/lib.rs
index 4df6c4b570..fd2ac58576 100644
--- a/datafusion/ffi/src/lib.rs
+++ b/datafusion/ffi/src/lib.rs
@@ -36,6 +36,7 @@ pub mod ffi_option;
pub mod insert_op;
pub mod physical_expr;
pub mod physical_optimizer;
+pub mod placement;
pub mod plan_properties;
pub mod proto;
pub mod record_batch_stream;
diff --git a/datafusion/ffi/src/placement.rs b/datafusion/ffi/src/placement.rs
new file mode 100644
index 0000000000..837f0e3aad
--- /dev/null
+++ b/datafusion/ffi/src/placement.rs
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion_expr::ExpressionPlacement;
+
+#[expect(non_camel_case_types)]
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FFI_ExpressionPlacement {
+ Literal,
+ Column,
+ MoveTowardsLeafNodes,
+ KeepInPlace,
+}
+
+impl From<ExpressionPlacement> for FFI_ExpressionPlacement {
+ fn from(value: ExpressionPlacement) -> Self {
+ match value {
+ ExpressionPlacement::Literal => Self::Literal,
+ ExpressionPlacement::Column => Self::Column,
+ ExpressionPlacement::MoveTowardsLeafNodes => Self::MoveTowardsLeafNodes,
+ ExpressionPlacement::KeepInPlace => Self::KeepInPlace,
+ }
+ }
+}
+
+impl From<FFI_ExpressionPlacement> for ExpressionPlacement {
+ fn from(value: FFI_ExpressionPlacement) -> Self {
+ match value {
+ FFI_ExpressionPlacement::Literal => Self::Literal,
+ FFI_ExpressionPlacement::Column => Self::Column,
+ FFI_ExpressionPlacement::MoveTowardsLeafNodes => Self::MoveTowardsLeafNodes,
+ FFI_ExpressionPlacement::KeepInPlace => Self::KeepInPlace,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use datafusion::logical_expr::ExpressionPlacement;
+
+ use super::FFI_ExpressionPlacement;
+
+ fn test_round_trip_placement(placement: ExpressionPlacement) {
+ let ffi_placement: FFI_ExpressionPlacement = placement.into();
+ let round_trip: ExpressionPlacement = ffi_placement.into();
+
+ assert_eq!(placement, round_trip);
+ }
+
+ #[test]
+ fn test_all_round_trip_placement() {
+ test_round_trip_placement(ExpressionPlacement::Literal);
+ test_round_trip_placement(ExpressionPlacement::Column);
+ test_round_trip_placement(ExpressionPlacement::MoveTowardsLeafNodes);
+ test_round_trip_placement(ExpressionPlacement::KeepInPlace);
+ }
+}
diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs
index 03b3a7ab24..dcd0910ecb 100644
--- a/datafusion/ffi/src/tests/mod.rs
+++ b/datafusion/ffi/src/tests/mod.rs
@@ -90,6 +90,8 @@ pub struct ForeignLibraryModule {
pub create_timezone_udf: extern "C" fn() -> FFI_ScalarUDF,
+ pub create_placement_udf: extern "C" fn() -> FFI_ScalarUDF,
+
pub create_table_function:
extern "C" fn(FFI_LogicalExtensionCodec) -> FFI_TableFunction,
@@ -251,6 +253,7 @@ pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {
create_scalar_udf: create_ffi_abs_func,
create_nullary_udf: create_ffi_random_func,
create_timezone_udf: udf_udaf_udwf::create_timezone_func,
+ create_placement_udf: udf_udaf_udwf::create_placement_func,
create_table_function: create_ffi_table_func,
create_sum_udaf: create_ffi_sum_func,
create_stddev_udaf: create_ffi_stddev_func,
diff --git a/datafusion/ffi/src/tests/udf_udaf_udwf.rs b/datafusion/ffi/src/tests/udf_udaf_udwf.rs
index 399a2cc6be..b393f5db3a 100644
--- a/datafusion/ffi/src/tests/udf_udaf_udwf.rs
+++ b/datafusion/ffi/src/tests/udf_udaf_udwf.rs
@@ -21,8 +21,8 @@ use arrow_schema::DataType;
use datafusion_catalog::TableFunctionImpl;
use datafusion_common::ScalarValue;
use datafusion_expr::{
- AggregateUDF, ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
- Volatility, WindowUDF,
+ AggregateUDF, ColumnarValue, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF,
+ ScalarUDFImpl, Signature, Volatility, WindowUDF,
};
use datafusion_functions::math::abs::AbsFunc;
use datafusion_functions::math::random::RandomFunc;
@@ -112,6 +112,56 @@ pub(crate) extern "C" fn create_timezone_func() -> FFI_ScalarUDF {
udf.into()
}
+#[derive(Debug, PartialEq, Eq, Hash)]
+struct PlacementUDF {
+ signature: Signature,
+}
+
+impl ScalarUDFImpl for PlacementUDF {
+ fn name(&self) -> &str {
+ "placement_udf"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(
+ &self,
+ _arg_types: &[DataType],
+ ) -> datafusion_common::Result<DataType> {
+ Ok(DataType::Int64)
+ }
+
+ fn invoke_with_args(
+ &self,
+ _args: ScalarFunctionArgs,
+ ) -> datafusion_common::Result<ColumnarValue> {
+ datafusion_common::internal_err!("placement_udf is not meant to be invoked")
+ }
+
+ fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {
+ // Push to the leaves only for a (Column, Literal) pairing, so the
+ // test catches dropped, reordered, or truncated arguments.
+ if matches!(
+ args,
+ [ExpressionPlacement::Column, ExpressionPlacement::Literal]
+ ) {
+ ExpressionPlacement::MoveTowardsLeafNodes
+ } else {
+ ExpressionPlacement::KeepInPlace
+ }
+ }
+}
+
+pub(crate) extern "C" fn create_placement_func() -> FFI_ScalarUDF {
+ let udf: Arc<ScalarUDF> = Arc::new(ScalarUDF::from(PlacementUDF {
+ signature: Signature::uniform(1, vec![DataType::Int64], Volatility::Immutable),
+ }));
+
+ udf.into()
+}
+
pub(crate) extern "C" fn create_ffi_table_func(
codec: FFI_LogicalExtensionCodec,
) -> FFI_TableFunction {
diff --git a/datafusion/ffi/src/udf/mod.rs b/datafusion/ffi/src/udf/mod.rs
index ff18a30e4b..4fc22e859f 100644
--- a/datafusion/ffi/src/udf/mod.rs
+++ b/datafusion/ffi/src/udf/mod.rs
@@ -28,8 +28,8 @@ use datafusion_common::config::ConfigOptions;
use datafusion_common::{DataFusionError, Result, internal_err};
use datafusion_expr::type_coercion::functions::fields_with_udf;
use datafusion_expr::{
- ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
- Signature,
+ ColumnarValue, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,
+ ScalarUDFImpl, Signature,
};
use return_type_args::{
FFI_ReturnFieldArgs, ForeignReturnFieldArgs, ForeignReturnFieldArgsOwned,
@@ -41,6 +41,7 @@ use stabby::vec::Vec as SVec;
use crate::arrow_wrappers::{WrappedArray, WrappedSchema};
use crate::config::FFI_ConfigOptions;
use crate::expr::columnar_value::FFI_ColumnarValue;
+use crate::placement::FFI_ExpressionPlacement;
use crate::util::{
FFI_Result, rvec_wrapped_to_vec_datatype, vec_datatype_to_rvec_wrapped,
};
@@ -91,6 +92,14 @@ pub struct FFI_ScalarUDF {
arg_types: SVec<WrappedSchema>,
) -> FFI_Result<SVec<WrappedSchema>>,
+ /// FFI equivalent to the `placement` of a [`ScalarUDFImpl`]. Returns the
+ /// placement hint for the underlying [`ScalarUDF`] given each argument's
+ /// placement. Infallible, so it returns the value directly, not an `FFI_Result`.
+ pub placement: unsafe extern "C" fn(
+ udf: &Self,
+ args: SVec<FFI_ExpressionPlacement>,
+ ) -> FFI_ExpressionPlacement,
+
/// Used to create a clone on the provider of the udf. This should
/// only need to be called by the receiver of the udf.
pub clone: unsafe extern "C" fn(udf: &Self) -> Self,
@@ -157,6 +166,18 @@ unsafe extern "C" fn coerce_types_fn_wrapper(
sresult!(vec_datatype_to_rvec_wrapped(&return_types))
}
+unsafe extern "C" fn placement_fn_wrapper(
+ udf: &FFI_ScalarUDF,
+ args: SVec<FFI_ExpressionPlacement>,
+) -> FFI_ExpressionPlacement {
+ let args = args
+ .into_iter()
+ .map(ExpressionPlacement::from)
+ .collect::<Vec<_>>();
+
+ udf.inner().placement(&args).into()
+}
+
unsafe extern "C" fn invoke_with_args_fn_wrapper(
udf: &FFI_ScalarUDF,
args: SVec<WrappedArray>,
@@ -250,6 +271,7 @@ impl From<Arc<ScalarUDF>> for FFI_ScalarUDF {
invoke_with_args: invoke_with_args_fn_wrapper,
return_field_from_args: return_field_from_args_fn_wrapper,
coerce_types: coerce_types_fn_wrapper,
+ placement: placement_fn_wrapper,
clone: clone_fn_wrapper,
release: release_fn_wrapper,
private_data: Box::into_raw(private_data) as *mut c_void,
@@ -427,12 +449,59 @@ impl ScalarUDFImpl for ForeignScalarUDF {
Ok(rvec_wrapped_to_vec_datatype(&result_types)?)
}
}
+
+ fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {
+ let args = args
+ .iter()
+ .map(|p| FFI_ExpressionPlacement::from(*p))
+ .collect::<SVec<_>>();
+
+ let result = unsafe { (self.udf.placement)(&self.udf, args) };
+
+ result.into()
+ }
}
#[cfg(test)]
mod tests {
use super::*;
+ #[derive(Debug, PartialEq, Eq, Hash)]
+ struct PlacementUDF {
+ signature: Signature,
+ }
+
+ impl ScalarUDFImpl for PlacementUDF {
+ fn name(&self) -> &str {
+ "placement_udf"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Int64)
+ }
+
+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ internal_err!("placement_udf is not meant to be invoked")
+ }
+
+ fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {
+ // Push to the leaves only for a (Column, Literal) pairing, so the
+ // test catches dropped, reordered, or truncated arguments.
+ if matches!(
+ args,
+ [ExpressionPlacement::Column, ExpressionPlacement::Literal]
+ ) {
+ ExpressionPlacement::MoveTowardsLeafNodes
+ } else {
+ ExpressionPlacement::KeepInPlace
+ }
+ }
+ }
+
#[test]
fn test_round_trip_scalar_udf() -> Result<()> {
let original_udf = datafusion::functions::math::abs::AbsFunc::new();
@@ -467,4 +536,42 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_ffi_udf_placement_round_trip() -> Result<()> {
+ use datafusion_expr::Volatility;
+
+ let original_udf = Arc::new(ScalarUDF::from(PlacementUDF {
+ signature: Signature::uniform(
+ 1,
+ vec![DataType::Int64],
+ Volatility::Immutable,
+ ),
+ }));
+
+ let mut ffi_udf = FFI_ScalarUDF::from(original_udf);
+
+ // Force the foreign path so the call travels through the FFI vtable
+ // rather than downcasting back to the original local type.
+ ffi_udf.library_marker_id = crate::mock_foreign_marker_id;
+ let foreign_udf: Arc<dyn ScalarUDFImpl> = (&ffi_udf).into();
+ assert!(foreign_udf.is::<ForeignScalarUDF>());
+
+ // Without the plumbing the override is dropped and every call is
+ // KeepInPlace. The three cases also check the arguments survive the
+ // round trip in order.
+ assert_eq!(
+ foreign_udf
+ .placement(&[ExpressionPlacement::Column, ExpressionPlacement::Literal]),
+ ExpressionPlacement::MoveTowardsLeafNodes
+ );
+ assert_eq!(
+ foreign_udf
+ .placement(&[ExpressionPlacement::Literal, ExpressionPlacement::Column]),
+ ExpressionPlacement::KeepInPlace
+ );
+ assert_eq!(foreign_udf.placement(&[]), ExpressionPlacement::KeepInPlace);
+
+ Ok(())
+ }
}
diff --git a/datafusion/ffi/tests/ffi_udf.rs b/datafusion/ffi/tests/ffi_udf.rs
index 6e6cb31f53..dffaf83c47 100644
--- a/datafusion/ffi/tests/ffi_udf.rs
+++ b/datafusion/ffi/tests/ffi_udf.rs
@@ -23,7 +23,7 @@ mod tests {
use arrow::datatypes::DataType;
use datafusion::common::record_batch;
use datafusion::error::Result;
- use datafusion::logical_expr::{ScalarUDF, ScalarUDFImpl};
+ use datafusion::logical_expr::{ExpressionPlacement, ScalarUDF, ScalarUDFImpl};
use datafusion::prelude::{SessionContext, col};
use datafusion_execution::config::SessionConfig;
use datafusion_expr::lit;
@@ -91,6 +91,31 @@ mod tests {
Ok(())
}
+ /// This test validates that a producer's `placement` override survives the
+ /// FFI boundary instead of collapsing to the default `KeepInPlace`.
+ #[tokio::test]
+ async fn test_scalar_udf_placement() -> Result<()> {
+ let module = get_module()?;
+
+ let ffi_placement_func = (module.create_placement_udf)();
+ let foreign_func: Arc<dyn ScalarUDFImpl> = (&ffi_placement_func).into();
+
+ // The override pushes to the leaves only for (Column, Literal), so these
+ // also check the arguments cross the boundary in order.
+ assert_eq!(
+ foreign_func
+ .placement(&[ExpressionPlacement::Column, ExpressionPlacement::Literal]),
+ ExpressionPlacement::MoveTowardsLeafNodes
+ );
+ assert_eq!(
+ foreign_func
+ .placement(&[ExpressionPlacement::Literal, ExpressionPlacement::Column]),
+ ExpressionPlacement::KeepInPlace
+ );
+
+ Ok(())
+ }
+
#[tokio::test]
async fn test_config_on_scalar_udf() -> Result<()> {
let module = get_module()?;