Skip to content

Apache DataFusion SQL Query Engine

minor: make HigherOrderSignature less error-prone (#22106)

gstvg557a4f09b21a
diff --git a/datafusion/expr/src/higher_order_function.rs b/datafusion/expr/src/higher_order_function.rs
index c84d5688a0..00522ad97b 100644
--- a/datafusion/expr/src/higher_order_function.rs
+++ b/datafusion/expr/src/higher_order_function.rs
@@ -91,8 +91,6 @@ pub struct HigherOrderSignature {
pub type_signature: HigherOrderTypeSignature,
/// The volatility of the function. See [Volatility] for more information.
pub volatility: Volatility,
- /// Whether [HigherOrderUDF::coerce_values_for_lambdas] should be called
- pub coerce_values_for_lambdas: bool,
/// The max number of times to call [HigherOrderUDF::lambda_parameters] before raising an error.
/// Used to guard against implementations that causes an infinite loop by endlessly returning
/// [LambdaParametersProgress::Partial]. Defaults to 256
@@ -107,7 +105,6 @@ impl HigherOrderSignature {
HigherOrderSignature {
type_signature,
volatility,
- coerce_values_for_lambdas: false,
lambda_parameters_max_iterations: LAMBDA_PARAMETERS_MAX_ITERATIONS,
}
}
@@ -117,7 +114,6 @@ impl HigherOrderSignature {
Self {
type_signature: HigherOrderTypeSignature::UserDefined,
volatility,
- coerce_values_for_lambdas: false,
lambda_parameters_max_iterations: LAMBDA_PARAMETERS_MAX_ITERATIONS,
}
}
@@ -127,7 +123,6 @@ impl HigherOrderSignature {
Self {
type_signature: HigherOrderTypeSignature::VariadicAny,
volatility,
- coerce_values_for_lambdas: false,
lambda_parameters_max_iterations: LAMBDA_PARAMETERS_MAX_ITERATIONS,
}
}
@@ -137,7 +132,6 @@ impl HigherOrderSignature {
Self {
type_signature: HigherOrderTypeSignature::Any(arg_count),
volatility,
- coerce_values_for_lambdas: false,
lambda_parameters_max_iterations: LAMBDA_PARAMETERS_MAX_ITERATIONS,
}
}
@@ -159,18 +153,9 @@ impl HigherOrderSignature {
Self {
type_signature: HigherOrderTypeSignature::Exact(args),
volatility,
- coerce_values_for_lambdas: false,
lambda_parameters_max_iterations: LAMBDA_PARAMETERS_MAX_ITERATIONS,
}
}
-
- /// Set [Self::coerce_values_for_lambdas] to true to indicate that [HigherOrderUDF::coerce_values_for_lambdas]
- /// should be called
- pub fn with_coerce_values_for_lambdas(mut self) -> Self {
- self.coerce_values_for_lambdas = true;
-
- self
- }
}
impl PartialEq for dyn HigherOrderUDF {
@@ -647,12 +632,12 @@ pub trait HigherOrderUDF: Debug + DynEq + DynHash + Send + Sync + Any {
///
/// assert_eq!(
/// coerce_to,
- /// vec![
+ /// Some(vec![
/// // return the same type for the array being reduced
/// DataType::new_list(DataType::Float32, true),
/// // coerce the initial value to the output of the merge lambda
/// DataType::Float32,
- /// ]
+ /// ])
/// );
///
/// ```
@@ -662,7 +647,7 @@ pub trait HigherOrderUDF: Debug + DynEq + DynHash + Send + Sync + Any {
///
/// The implementation can assume that some other part of the code has coerced
/// the actual argument types to match [`Self::signature`], except the coercion defined by
- /// [Self::coerce_values_for_lambdas], if applicable.
+ /// [Self::coerce_values_for_lambdas].
///
/// [`HigherOrderFunction`]: crate::expr::HigherOrderFunction
/// [`HigherOrderFunction::lambda_parameters`]: crate::expr::HigherOrderFunction::lambda_parameters
@@ -675,8 +660,7 @@ pub trait HigherOrderUDF: Debug + DynEq + DynHash + Send + Sync + Any {
/// Coerce value arguments of a function call to types that the function can evaluate also taking into
/// account the *output type of it's lambdas*. This differs from [HigherOrderUDF::coerce_value_types]
/// that only has access to the type of it's value arguments because it's called before the output type
- /// of lambdas are known. So that this method is called, the function must have it's
- /// [HigherOrderSignature::coerce_values_for_lambdas] set to true
+ /// of lambdas are known.
///
/// See the [type coercion module](crate::type_coercion)
/// documentation for more details on type coercion
@@ -685,29 +669,27 @@ pub trait HigherOrderUDF: Debug + DynEq + DynHash + Send + Sync + Any {
/// * `fields`: The argument types of the value arguments of this function, or the output type of lambdas
///
/// # Return value
- /// A Vec with the same number of [ValueOrLambda::Value] in `fields`. DataFusion will `CAST` the
- /// function call arguments to these specific types.
+ /// If `Some`, contains a Vec with the same number of [ValueOrLambda::Value] in `fields`.
+ /// DataFusion will `CAST` the function call arguments to these specific types. If `None`, no
+ /// coercion will be applied beyond the one defined by the function signature.
///
/// For example, a flexible array_reduce implementation (see [Self::lambda_parameters] docs), when working
/// with the expression below, may want to coerce it's initial value argument, the *integer* `0`,
- /// to match the output it's merge function, which is a *float*:
+ /// to match the output of it's merge function, which is a *float*:
///
/// `array_reduce([1.2, 2.1], 0, (acc, v) -> acc + v + 1.5, v -> v > 2.0)`
fn coerce_values_for_lambdas(
&self,
_fields: &[ValueOrLambda<DataType, DataType>],
- ) -> Result<Vec<DataType>> {
- not_impl_err!(
- "{} coerce_values_for_lambdas is not implemented",
- self.name()
- )
+ ) -> Result<Option<Vec<DataType>>> {
+ Ok(None)
}
/// What type will be returned by this function, given the arguments?
///
/// The implementation can assume that some other part of the code has coerced
/// the actual argument types to match [`Self::signature`], including the coercion
- /// defined by [Self::coerce_values_for_lambdas], if applicable.
+ /// defined by [Self::coerce_values_for_lambdas].
///
/// # Example creating `Field`
///
diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs
index 624f0da3fd..1f625e33d3 100644
--- a/datafusion/expr/src/type_coercion/functions.rs
+++ b/datafusion/expr/src/type_coercion/functions.rs
@@ -158,9 +158,9 @@ pub fn fields_with_udf<F: UDFCoercionExt>(
/// argument must be coerced to match `signature`.
/// For lambda arguments, returns a clone of the associated data
///
-/// Note this does not invokes [HigherOrderUDF::coerce_values_for_lambdas]
-/// if requested by the function signature. If that's required, use
-/// [value_fields_with_higher_order_udf_and_lambdas] instead
+/// Note this does not invokes [HigherOrderUDF::coerce_values_for_lambdas].
+/// If that's required, use [value_fields_with_higher_order_udf_and_lambdas]
+/// instead
///
/// For more details on coercion in general, please see the
/// [`type_coercion`](crate::type_coercion) module.
@@ -307,8 +307,8 @@ pub fn value_fields_with_higher_order_udf<L: Clone>(
/// Performs type coercion for higher order function arguments,
/// including those defined by [HigherOrderUDF::coerce_values_for_lambdas],
-/// if defined by the signature. Note that compared to
-/// [value_fields_with_higher_order_udf], this function requires
+/// if it returns `Some(...)` instead of the default `None`. Note that
+/// compared to [value_fields_with_higher_order_udf], this function requires
/// the [ValueOrLambda::Lambda] variant to contain the output field of the lambda.
///
/// For value arguments, returns the field to which each
@@ -323,16 +323,16 @@ pub fn value_fields_with_higher_order_udf_and_lambdas(
) -> Result<Vec<ValueOrLambda<FieldRef, FieldRef>>> {
let mut new_fields = value_fields_with_higher_order_udf(current_fields, func)?;
- if func.signature().coerce_values_for_lambdas {
- let new_types = new_fields
- .iter()
- .map(|f| match f {
- ValueOrLambda::Value(f) => ValueOrLambda::Value(f.data_type().clone()),
- ValueOrLambda::Lambda(f) => ValueOrLambda::Lambda(f.data_type().clone()),
- })
- .collect::<Vec<_>>();
+ let new_types = new_fields
+ .iter()
+ .map(|f| match f {
+ ValueOrLambda::Value(f) => ValueOrLambda::Value(f.data_type().clone()),
+ ValueOrLambda::Lambda(f) => ValueOrLambda::Lambda(f.data_type().clone()),
+ })
+ .collect::<Vec<_>>();
- let mut new_value_types = func.coerce_values_for_lambdas(&new_types)?.into_iter();
+ if let Some(new_value_types) = func.coerce_values_for_lambdas(&new_types)? {
+ let mut new_value_types = new_value_types.into_iter();
let value_types_count = new_types
.iter()
@@ -1923,7 +1923,7 @@ mod tests {
fn coerce_values_for_lambdas(
&self,
fields: &[ValueOrLambda<DataType, DataType>],
- ) -> Result<Vec<DataType>> {
+ ) -> Result<Option<Vec<DataType>>> {
// thoerical impl of array_reduce without finish
let [
ValueOrLambda::Value(list),
@@ -1934,7 +1934,7 @@ mod tests {
unreachable!()
};
- Ok(vec![list.clone(), merge.clone()])
+ Ok(Some(vec![list.clone(), merge.clone()]))
}
fn lambda_parameters(
@@ -1997,8 +1997,7 @@ mod tests {
#[test]
fn test_higher_order_function_coerce_values_for_lambdas() {
let fun = MockHigherOrderUDF {
- signature: HigherOrderSignature::variadic_any(Volatility::Immutable)
- .with_coerce_values_for_lambdas(),
+ signature: HigherOrderSignature::variadic_any(Volatility::Immutable),
coerced_value_types: vec![],
};