Skip to content

Apache DataFusion SQL Query Engine

fix: Emit LIKE the way the Substrait extension defines it (#25442)

namanjain24-sudof868d604b658
diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs
index 13f86997f3..c688b25de2 100644
--- a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs
@@ -187,6 +187,32 @@ fn arg_list_to_binary_op_tree_inner(
}))
}
+/// Reads the `case_sensitivity` option of a `like` call.
+///
+/// Substrait says a consumer must use the first value it supports, and must
+/// reject the call when it supports none of them.
+fn case_insensitive_option(f: &ScalarFunction) -> Result<bool> {
+ let Some(option) = f
+ .options
+ .iter()
+ .find(|option| option.name.eq_ignore_ascii_case("case_sensitivity"))
+ else {
+ return Ok(false);
+ };
+ for preference in &option.preference {
+ if preference.eq_ignore_ascii_case("CASE_SENSITIVE") {
+ return Ok(false);
+ }
+ if preference.eq_ignore_ascii_case("CASE_INSENSITIVE") {
+ return Ok(true);
+ }
+ }
+ not_impl_err!(
+ "Unsupported case_sensitivity for `like`: {:?}",
+ option.preference
+ )
+}
+
/// Build [`Expr`] from its name and required inputs.
struct BuiltinExprBuilder {
expr_name: String,
@@ -213,7 +239,10 @@ impl BuiltinExprBuilder {
args: Vec<Expr>,
) -> Result<Expr> {
match self.expr_name.as_str() {
- "like" => Self::build_like_expr(false, false, f, args),
+ // `like` carries case sensitivity as an option. `ilike` is not a
+ // Substrait function, but DataFusion used to emit it, so plans
+ // written by an older version are still read.
+ "like" => Self::build_like_expr(case_insensitive_option(f)?, false, f, args),
"ilike" => Self::build_like_expr(true, false, f, args),
"like_match" => Self::build_like_expr(false, false, f, args),
"like_imatch" => Self::build_like_expr(true, false, f, args),
@@ -374,7 +403,7 @@ impl BuiltinExprBuilder {
#[cfg(test)]
mod tests {
- use super::arg_list_to_binary_op_tree;
+ use super::{arg_list_to_binary_op_tree, case_insensitive_option};
use crate::extensions::Extensions;
use crate::logical_plan::consumer::tests::TEST_SESSION_STATE;
use crate::logical_plan::consumer::{DefaultSubstraitConsumer, SubstraitConsumer};
@@ -583,4 +612,83 @@ mod tests {
Ok(())
}
+
+ fn scalar_function_with_case_sensitivity_preference(
+ preference: &[&str],
+ ) -> ScalarFunction {
+ ScalarFunction {
+ options: vec![substrait::proto::FunctionOption {
+ name: "case_sensitivity".to_string(),
+ preference: preference.iter().map(|s| s.to_string()).collect(),
+ }],
+ ..Default::default()
+ }
+ }
+
+ /// Substrait says a consumer must use the first value it supports in the
+ /// `case_sensitivity` option's preference list, and must reject the call
+ /// when it supports none of them.
+ #[test]
+ fn case_insensitive_option_cases() -> Result<()> {
+ // No `case_sensitivity` option at all defaults to case-sensitive.
+ assert!(!case_insensitive_option(&ScalarFunction::default())?);
+
+ // A single supported preference, in either sensitivity.
+ assert!(!case_insensitive_option(
+ &scalar_function_with_case_sensitivity_preference(&["CASE_SENSITIVE"])
+ )?);
+ assert!(case_insensitive_option(
+ &scalar_function_with_case_sensitivity_preference(&["CASE_INSENSITIVE"])
+ )?);
+
+ // The option name and its preference values are matched
+ // case-insensitively (ASCII).
+ assert!(case_insensitive_option(&ScalarFunction {
+ options: vec![substrait::proto::FunctionOption {
+ name: "Case_Sensitivity".to_string(),
+ preference: vec!["case_insensitive".to_string()],
+ }],
+ ..Default::default()
+ })?);
+
+ // Preference ordering: a consumer must use the first value it
+ // supports, not necessarily the first value in the list.
+ // `CASE_INSENSITIVE_ASCII` is not a value this consumer supports, so
+ // it is skipped in favor of the next, supported preference.
+ assert!(case_insensitive_option(
+ &scalar_function_with_case_sensitivity_preference(&[
+ "CASE_INSENSITIVE_ASCII",
+ "CASE_INSENSITIVE"
+ ])
+ )?);
+ assert!(!case_insensitive_option(
+ &scalar_function_with_case_sensitivity_preference(&[
+ "CASE_INSENSITIVE_ASCII",
+ "CASE_SENSITIVE"
+ ])
+ )?);
+
+ // Rejection: every preference is unsupported.
+ let err =
+ case_insensitive_option(&scalar_function_with_case_sensitivity_preference(
+ &["CASE_INSENSITIVE_ASCII"],
+ ))
+ .unwrap_err();
+ assert!(
+ err.to_string().contains("Unsupported case_sensitivity"),
+ "unexpected error: {err}"
+ );
+
+ // Rejection: an empty preference list supports nothing.
+ let err = case_insensitive_option(
+ &scalar_function_with_case_sensitivity_preference(&[]),
+ )
+ .unwrap_err();
+ assert!(
+ err.to_string().contains("Unsupported case_sensitivity"),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+ }
}
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs
index 8deee8b657..2f4fb5c1ce 100644
--- a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs
+++ b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs
@@ -16,17 +16,17 @@
// under the License.
use crate::logical_plan::producer::{
- SubstraitProducer, to_substrait_literal_expr, to_substrait_type,
- to_substrait_type_from_field,
+ SubstraitProducer, to_substrait_type, to_substrait_type_from_field,
};
use datafusion::arrow::datatypes::DataType;
use datafusion::common::datatype::FieldExt;
use datafusion::common::{
- DFSchemaRef, ScalarValue, internal_datafusion_err, not_impl_err, substrait_err,
+ DFSchemaRef, internal_datafusion_err, not_impl_err, substrait_err,
};
use datafusion::logical_expr::{
Between, BinaryExpr, Expr, ExprSchemable, Like, Operator, expr,
};
+use substrait::proto::FunctionOption;
use substrait::proto::expression::{RexType, ScalarFunction};
use substrait::proto::function_argument::ArgType;
use substrait::proto::{Expression, FunctionArgument, Type};
@@ -234,6 +234,11 @@ pub fn from_binary_expr(
))
}
+/// The option `like` uses to carry case sensitivity, and the value that asks
+/// for the case insensitive form. Defined in `functions_string.yaml`.
+pub(crate) const CASE_SENSITIVITY_OPTION: &str = "case_sensitivity";
+pub(crate) const CASE_INSENSITIVE: &str = "CASE_INSENSITIVE";
+
pub fn from_like(
producer: &mut impl SubstraitProducer,
like: &Like,
@@ -246,11 +251,14 @@ pub fn from_like(
escape_char,
case_insensitive,
} = like;
- let function_anchor = if *case_insensitive {
- producer.register_function("ilike".to_string())
- } else {
- producer.register_function("like".to_string())
- };
+ // `like` takes two arguments and carries case sensitivity as an option;
+ // the extensions define no `ilike` and no escape character, so an escape
+ // has no encoding here and is rejected rather than emitted as a third
+ // argument that a consumer would bind to a parameter the function does
+ // not have.
+ if escape_char.is_some() {
+ return not_impl_err!("Substrait does not define an escape character for `like`");
+ }
// Substrait documents `output_type` as "Must be set to the return type of
// the function, exactly as derived using the declaration in the extension",
// and a consumer that reads it rejects the call when it is unset. The type
@@ -258,23 +266,27 @@ pub fn from_like(
// derives, rather than being restated here.
let (_, output_field) = Expr::Like(like.clone()).to_field(schema)?;
let output_type = to_substrait_type_from_field(producer, &output_field)?;
- let expr = producer.handle_expr(expr, schema)?;
- let pattern = producer.handle_expr(pattern, schema)?;
- let escape_char = to_substrait_literal_expr(
- producer,
- &ScalarValue::Utf8(escape_char.map(|c| c.to_string())),
- )?;
+
+ let function_anchor = producer.register_function("like".to_string());
+ let substrait_expr = producer.handle_expr(expr, schema)?;
+ let substrait_pattern = producer.handle_expr(pattern, schema)?;
let arguments = vec![
FunctionArgument {
- arg_type: Some(ArgType::Value(expr)),
- },
- FunctionArgument {
- arg_type: Some(ArgType::Value(pattern)),
+ arg_type: Some(ArgType::Value(substrait_expr)),
},
FunctionArgument {
- arg_type: Some(ArgType::Value(escape_char)),
+ arg_type: Some(ArgType::Value(substrait_pattern)),
},
];
+ // An unset option leaves the default, which is `CASE_SENSITIVE`.
+ let options = if *case_insensitive {
+ vec![FunctionOption {
+ name: CASE_SENSITIVITY_OPTION.to_string(),
+ preference: vec![CASE_INSENSITIVE.to_string()],
+ }]
+ } else {
+ vec![]
+ };
#[expect(deprecated)]
let substrait_like = Expression {
@@ -283,7 +295,7 @@ pub fn from_like(
arguments,
output_type: Some(output_type.clone()),
args: vec![],
- options: vec![],
+ options,
})),
};
@@ -438,6 +450,7 @@ pub fn operator_to_name(op: Operator) -> &'static str {
#[cfg(test)]
mod tests {
+ use super::{CASE_INSENSITIVE, CASE_SENSITIVITY_OPTION};
use crate::logical_plan::producer::{
DefaultSubstraitProducer, SubstraitProducer, to_substrait_type,
};
@@ -447,9 +460,88 @@ mod tests {
use datafusion::logical_expr::{Expr, Like};
use datafusion::prelude::{col, lit};
use substrait::proto::Expression;
+ use substrait::proto::FunctionOption;
use substrait::proto::expression::{RexType, ScalarFunction};
use substrait::proto::function_argument::ArgType;
+ /// `like` takes two arguments and carries case sensitivity as an option,
+ /// so `ILIKE` is that option rather than a separate function.
+ #[tokio::test]
+ async fn like_emits_case_sensitivity_option() -> datafusion::common::Result<()> {
+ let state = SessionStateBuilder::default().build();
+ let schema =
+ DFSchemaRef::new(DFSchema::try_from(Schema::new(vec![Field::new(
+ "s",
+ DataType::Utf8,
+ true,
+ )]))?);
+
+ for (case_insensitive, expected_options) in [
+ (false, vec![]),
+ (
+ true,
+ vec![FunctionOption {
+ name: CASE_SENSITIVITY_OPTION.to_string(),
+ preference: vec![CASE_INSENSITIVE.to_string()],
+ }],
+ ),
+ ] {
+ let mut producer = DefaultSubstraitProducer::new(&state);
+ let like = Like::new(
+ false,
+ Box::new(col("s")),
+ Box::new(lit("a%")),
+ None,
+ case_insensitive,
+ );
+ let expr = producer.handle_expr(&Expr::Like(like), &schema)?;
+
+ let Some(RexType::ScalarFunction(call)) = expr.rex_type else {
+ panic!("Substrait ScalarFunction expected")
+ };
+ assert_eq!(
+ producer
+ .get_extensions()
+ .functions
+ .get(&call.function_reference),
+ Some(&"like".to_string()),
+ "case_insensitive = {case_insensitive}"
+ );
+ assert_eq!(call.arguments.len(), 2, "no escape argument is emitted");
+ assert_eq!(call.options, expected_options);
+ }
+
+ Ok(())
+ }
+
+ /// The extensions define no escape character, so there is nothing to emit.
+ #[tokio::test]
+ async fn like_with_escape_is_rejected() -> datafusion::common::Result<()> {
+ let state = SessionStateBuilder::default().build();
+ let schema =
+ DFSchemaRef::new(DFSchema::try_from(Schema::new(vec![Field::new(
+ "s",
+ DataType::Utf8,
+ true,
+ )]))?);
+ let mut producer = DefaultSubstraitProducer::new(&state);
+
+ let like = Like::new(
+ false,
+ Box::new(col("s")),
+ Box::new(lit("a!%")),
+ Some('!'),
+ false,
+ );
+ let err = producer
+ .handle_expr(&Expr::Like(like), &schema)
+ .expect_err("an escape character must be rejected")
+ .to_string();
+ assert!(err.contains("escape character"), "unexpected error: {err}");
+
+ Ok(())
+ }
+
#[tokio::test]
async fn binary_expr_output_type() -> datafusion::common::Result<()> {
let state = SessionStateBuilder::default().build();