Skip to content

Apache DataFusion SQL Query Engine

fix: Set Substrait output_type on window functions and LIKE (#25367)

namanjain24-sudo62f29430d262
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs
index 75720395aa..8deee8b657 100644
--- a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs
+++ b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs
@@ -17,6 +17,7 @@
use crate::logical_plan::producer::{
SubstraitProducer, to_substrait_literal_expr, to_substrait_type,
+ to_substrait_type_from_field,
};
use datafusion::arrow::datatypes::DataType;
use datafusion::common::datatype::FieldExt;
@@ -245,31 +246,18 @@ pub fn from_like(
escape_char,
case_insensitive,
} = like;
- make_substrait_like_expr(
- producer,
- *case_insensitive,
- *negated,
- expr,
- pattern,
- *escape_char,
- schema,
- )
-}
-
-fn make_substrait_like_expr(
- producer: &mut impl SubstraitProducer,
- ignore_case: bool,
- negated: bool,
- expr: &Expr,
- pattern: &Expr,
- escape_char: Option<char>,
- schema: &DFSchemaRef,
-) -> datafusion::common::Result<Expression> {
- let function_anchor = if ignore_case {
+ let function_anchor = if *case_insensitive {
producer.register_function("ilike".to_string())
} else {
producer.register_function("like".to_string())
};
+ // Substrait documents `output_type` as "Must be set to the return type of
+ // the function, exactly as derived using the declaration in the extension",
+ // and a consumer that reads it rejects the call when it is unset. The type
+ // comes from the expression itself so that it matches what DataFusion
+ // derives, rather than being restated here.
+ let (_, output_field) = Expr::Like(like.clone()).to_field(schema)?;
+ let output_type = to_substrait_type_from_field(producer, &output_field)?;
let expr = producer.handle_expr(expr, schema)?;
let pattern = producer.handle_expr(pattern, schema)?;
let escape_char = to_substrait_literal_expr(
@@ -293,13 +281,13 @@ fn make_substrait_like_expr(
rex_type: Some(RexType::ScalarFunction(ScalarFunction {
function_reference: function_anchor,
arguments,
- output_type: None,
+ output_type: Some(output_type.clone()),
args: vec![],
options: vec![],
})),
};
- if negated {
+ if *negated {
let function_anchor = producer.register_function("not".to_string());
#[expect(deprecated)]
@@ -309,7 +297,8 @@ fn make_substrait_like_expr(
arguments: vec![FunctionArgument {
arg_type: Some(ArgType::Value(substrait_like)),
}],
- output_type: None,
+ // `not` yields the type its argument does.
+ output_type: Some(output_type),
args: vec![],
options: vec![],
})),
@@ -452,12 +441,14 @@ mod tests {
use crate::logical_plan::producer::{
DefaultSubstraitProducer, SubstraitProducer, to_substrait_type,
};
- use datafusion::arrow::datatypes::DataType;
+ use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::common::{DFSchema, DFSchemaRef};
use datafusion::execution::SessionStateBuilder;
- use datafusion::prelude::lit;
+ use datafusion::logical_expr::{Expr, Like};
+ use datafusion::prelude::{col, lit};
use substrait::proto::Expression;
use substrait::proto::expression::{RexType, ScalarFunction};
+ use substrait::proto::function_argument::ArgType;
#[tokio::test]
async fn binary_expr_output_type() -> datafusion::common::Result<()> {
@@ -479,4 +470,45 @@ mod tests {
panic!("Substrait ScalarFunction expected")
}
}
+
+ /// The `like` call carries the type the expression yields, and so does the
+ /// `not` that wraps a negated one.
+ #[tokio::test]
+ async fn like_output_type() -> datafusion::common::Result<()> {
+ let state = SessionStateBuilder::default().build();
+ let schema =
+ DFSchemaRef::new(DFSchema::try_from(Schema::new(vec![Field::new(
+ "s",
+ DataType::Utf8,
+ true,
+ )]))?);
+ let mut producer = DefaultSubstraitProducer::new(&state);
+ // A `LIKE` over a nullable input yields a nullable boolean.
+ let expected = to_substrait_type(&mut producer, &DataType::Boolean, true)?;
+
+ let like = Like::new(false, Box::new(col("s")), Box::new(lit("a%")), None, false);
+ let substrait_expr = producer.handle_expr(&Expr::Like(like), &schema)?;
+ let Some(RexType::ScalarFunction(like_fn)) = substrait_expr.rex_type else {
+ panic!("Substrait ScalarFunction expected")
+ };
+ assert_eq!(like_fn.output_type, Some(expected.clone()));
+
+ let negated =
+ Like::new(true, Box::new(col("s")), Box::new(lit("a%")), None, false);
+ let substrait_expr = producer.handle_expr(&Expr::Like(negated), &schema)?;
+ let Some(RexType::ScalarFunction(not_fn)) = substrait_expr.rex_type else {
+ panic!("Substrait ScalarFunction expected")
+ };
+ assert_eq!(not_fn.output_type, Some(expected.clone()));
+
+ // The `not` wraps the `like`, which carries the type as well.
+ let Some(ArgType::Value(Expression {
+ rex_type: Some(RexType::ScalarFunction(inner)),
+ })) = not_fn.arguments[0].arg_type.clone()
+ else {
+ panic!("Substrait ScalarFunction expected inside `not`")
+ };
+ assert_eq!(inner.output_type, Some(expected));
+ Ok(())
+ }
}
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/window_function.rs b/datafusion/substrait/src/logical_plan/producer/expr/window_function.rs
index d35771bf09..98c8569c74 100644
--- a/datafusion/substrait/src/logical_plan/producer/expr/window_function.rs
+++ b/datafusion/substrait/src/logical_plan/producer/expr/window_function.rs
@@ -16,9 +16,11 @@
// under the License.
use crate::logical_plan::producer::SubstraitProducer;
+use crate::logical_plan::producer::to_substrait_type_from_field;
use crate::logical_plan::producer::utils::substrait_sort_field;
use datafusion::common::{DFSchemaRef, ScalarValue, not_impl_err};
use datafusion::logical_expr::expr::{WindowFunction, WindowFunctionParams};
+use datafusion::logical_expr::{Expr, ExprSchemable};
use datafusion::logical_expr::{WindowFrame, WindowFrameBound, WindowFrameUnits};
use substrait::proto::aggregate_function::AggregationInvocation;
use substrait::proto::expression::RexType;
@@ -27,7 +29,7 @@ use substrait::proto::expression::window_function::bound as SubstraitBound;
use substrait::proto::expression::window_function::bound::Kind as BoundKind;
use substrait::proto::expression::window_function::{Bound, BoundsType};
use substrait::proto::function_argument::ArgType;
-use substrait::proto::{Expression, FunctionArgument, SortField};
+use substrait::proto::{Expression, FunctionArgument};
pub fn from_window_function(
producer: &mut impl SubstraitProducer,
@@ -79,37 +81,26 @@ pub fn from_window_function(
// window frame
let bounds = to_substrait_bounds(window_frame)?;
let bound_type = to_substrait_bound_type(window_frame)?;
- Ok(make_substrait_window_function(
- function_anchor,
- arguments,
- partition_by,
- order_by,
- bounds,
- bound_type,
- *distinct,
- ))
-}
+ // Substrait documents `output_type` as "Must be set to the return type of
+ // the function, exactly as derived using the declaration in the extension",
+ // and a consumer that reads it rejects the call when it is unset. The type
+ // comes from the expression itself so that it matches what DataFusion
+ // derives, rather than being restated here.
+ let (_, output_field) =
+ Expr::WindowFunction(Box::new(window_fn.clone())).to_field(schema)?;
+ let output_type = to_substrait_type_from_field(producer, &output_field)?;
-fn make_substrait_window_function(
- function_reference: u32,
- arguments: Vec<FunctionArgument>,
- partitions: Vec<Expression>,
- sorts: Vec<SortField>,
- bounds: (Bound, Bound),
- bounds_type: BoundsType,
- distinct: bool,
-) -> Expression {
#[expect(deprecated)]
- Expression {
+ Ok(Expression {
rex_type: Some(RexType::WindowFunction(SubstraitWindowFunction {
- function_reference,
+ function_reference: function_anchor,
arguments,
- partitions,
- sorts,
+ partitions: partition_by,
+ sorts: order_by,
options: vec![],
- output_type: None,
+ output_type: Some(output_type),
phase: 0, // default to AGGREGATION_PHASE_UNSPECIFIED
- invocation: if distinct {
+ invocation: if *distinct {
AggregationInvocation::Distinct as i32
} else {
AggregationInvocation::All as i32
@@ -117,9 +108,9 @@ fn make_substrait_window_function(
lower_bound: Some(bounds.0),
upper_bound: Some(bounds.1),
args: vec![],
- bounds_type: bounds_type as i32,
+ bounds_type: bound_type as i32,
})),
- }
+ })
}
fn to_substrait_bound_type(
@@ -185,7 +176,67 @@ fn to_substrait_bound_offset(value: &ScalarValue) -> datafusion::common::Result<
#[cfg(test)]
mod tests {
use super::*;
- use datafusion::common::assert_contains;
+ use crate::logical_plan::producer::{DefaultSubstraitProducer, to_substrait_type};
+ use datafusion::arrow::datatypes::{DataType, Field, Schema};
+ use datafusion::common::{DFSchema, assert_contains};
+ use datafusion::execution::SessionStateBuilder;
+ use datafusion::functions_aggregate::count::count_all_window;
+ use datafusion::functions_aggregate::sum::sum_udaf;
+ use datafusion::logical_expr::expr::WindowFunction;
+ use datafusion::prelude::col;
+ use substrait::proto::expression::RexType;
+
+ /// Substrait requires the return type on a window function call, and a
+ /// consumer that reads it rejects the call when it is unset.
+ #[test]
+ fn window_function_output_type() -> datafusion::common::Result<()> {
+ let state = SessionStateBuilder::default().build();
+ let schema =
+ DFSchemaRef::new(DFSchema::try_from(Schema::new(vec![Field::new(
+ "i",
+ DataType::Int64,
+ true,
+ )]))?);
+ let mut producer = DefaultSubstraitProducer::new(&state);
+
+ let window_fn = WindowFunction::new(sum_udaf(), vec![col("i")]);
+ let expr = Expr::WindowFunction(Box::new(window_fn));
+ let substrait_expr = producer.handle_expr(&expr, &schema)?;
+
+ let Some(RexType::WindowFunction(window)) = substrait_expr.rex_type else {
+ panic!("Substrait WindowFunction expected")
+ };
+ // `sum` over a nullable i64 yields a nullable i64.
+ let expected = to_substrait_type(&mut producer, &DataType::Int64, true)?;
+ assert_eq!(window.output_type, Some(expected));
+ Ok(())
+ }
+
+ /// `COUNT(*) OVER ()` always yields a non-null count, so this guards
+ /// against a regression where `output_type` is emitted as nullable
+ /// unconditionally.
+ #[test]
+ fn count_window_function_output_type_is_non_nullable()
+ -> datafusion::common::Result<()> {
+ let state = SessionStateBuilder::default().build();
+ let schema =
+ DFSchemaRef::new(DFSchema::try_from(Schema::new(vec![Field::new(
+ "i",
+ DataType::Int64,
+ true,
+ )]))?);
+ let mut producer = DefaultSubstraitProducer::new(&state);
+
+ let substrait_expr = producer.handle_expr(&count_all_window(), &schema)?;
+
+ let Some(RexType::WindowFunction(window)) = substrait_expr.rex_type else {
+ panic!("Substrait WindowFunction expected")
+ };
+ // `count(*)` never produces a null, regardless of input nullability.
+ let expected = to_substrait_type(&mut producer, &DataType::Int64, false)?;
+ assert_eq!(window.output_type, Some(expected));
+ Ok(())
+ }
#[test]
fn window_frame_offsets() {