Apache DataFusion SQL Query Engine
feat: fix matching for named parameters with non-lowercase signatures (#19378)
bubulalabua95c7fc2c4bf
diff --git a/datafusion/expr/src/arguments.rs b/datafusion/expr/src/arguments.rsindex 6bd6e57236..f10cf50f60 100644--- a/datafusion/expr/src/arguments.rs+++ b/datafusion/expr/src/arguments.rs@@ -19,7 +19,20 @@use crate::Expr;use datafusion_common::{Result, plan_err};-use std::collections::HashMap;++/// Represents a named function argument with its original case and quote information.+///+/// This struct preserves whether an identifier was quoted in the SQL, which determines+/// whether case-sensitive or case-insensitive matching should be used per SQL standards.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ArgumentName {+ /// The argument name in its original case as it appeared in the SQL+ pub value: String,+ /// Whether the identifier was quoted (e.g., "STR" vs STR)+ /// - true: quoted identifier, requires case-sensitive matching+ /// - false: unquoted identifier, uses case-insensitive matching+ pub is_quoted: bool,+}/// Resolves function arguments, handling named and positional notation.///@@ -50,7 +63,7 @@ use std::collections::HashMap;pub fn resolve_function_arguments(param_names: &[String],args: Vec<Expr>,- arg_names: Vec<Option<String>>,+ arg_names: Vec<Option<ArgumentName>>,) -> Result<Vec<Expr>> {if args.len() != arg_names.len() {return plan_err!(@@ -71,7 +84,7 @@ pub fn resolve_function_arguments(}/// Validates that positional arguments come before named arguments-fn validate_argument_order(arg_names: &[Option<String>]) -> Result<()> {+fn validate_argument_order(arg_names: &[Option<ArgumentName>]) -> Result<()> {let mut seen_named = false;for (i, arg_name) in arg_names.iter().enumerate() {match arg_name {@@ -93,15 +106,8 @@ fn validate_argument_order(arg_names: &[Option<String>]) -> Result<()> {fn reorder_named_arguments(param_names: &[String],args: Vec<Expr>,- arg_names: Vec<Option<String>>,+ arg_names: Vec<Option<ArgumentName>>,) -> Result<Vec<Expr>> {- // Build HashMap for O(1) parameter name lookups- let param_index_map: HashMap<&str, usize> = param_names- .iter()- .enumerate()- .map(|(idx, name)| (name.as_str(), idx))- .collect();-let positional_count = arg_names.iter().filter(|n| n.is_none()).count();// Capture args length before consuming the vector@@ -120,19 +126,35 @@ fn reorder_named_arguments(let mut result: Vec<Option<Expr>> = vec![None; expected_arg_count];for (i, (arg, arg_name)) in args.into_iter().zip(arg_names).enumerate() {- if let Some(name) = arg_name {- // Named argument - O(1) lookup in HashMap- let param_index =- param_index_map.get(name.as_str()).copied().ok_or_else(|| {+ if let Some(arg_name) = arg_name {+ // Named argument - find parameter index using linear search+ // Match based on SQL identifier rules:+ // - Quoted identifiers: case-sensitive (exact match)+ // - Unquoted identifiers: case-insensitive match+ let param_index = param_names+ .iter()+ .position(|p| {+ if arg_name.is_quoted {+ // Quoted: exact case match+ p == &arg_name.value+ } else {+ // Unquoted: case-insensitive match+ p.eq_ignore_ascii_case(&arg_name.value)+ }+ })+ .ok_or_else(|| {datafusion_common::plan_datafusion_err!("Unknown parameter name '{}'. Valid parameters are: [{}]",- name,+ arg_name.value,param_names.join(", "))})?;if result[param_index].is_some() {- return plan_err!("Parameter '{}' specified multiple times", name);+ return plan_err!(+ "Parameter '{}' specified multiple times",+ arg_name.value+ );}result[param_index] = Some(arg);@@ -175,10 +197,109 @@ mod tests {let param_names = vec!["a".to_string(), "b".to_string()];let args = vec![lit(1), lit("hello")];- let arg_names = vec![Some("a".to_string()), Some("b".to_string())];+ let arg_names = vec![+ Some(ArgumentName {+ value: "a".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "b".to_string(),+ is_quoted: false,+ }),+ ];++ let result = resolve_function_arguments(¶m_names, args, arg_names).unwrap();+ assert_eq!(result.len(), 2);+ }++ #[test]+ fn test_case_insensitive_parameter_matching() {+ // Parameter names in function signature (lowercase)+ let param_names = vec!["startpos".to_string(), "length".to_string()];++ // Unquoted arguments with different casing should match case-insensitively+ let args = vec![lit(1), lit(10)];+ let arg_names = vec![+ Some(ArgumentName {+ value: "STARTPOS".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "LENGTH".to_string(),+ is_quoted: false,+ }),+ ];let result = resolve_function_arguments(¶m_names, args, arg_names).unwrap();assert_eq!(result.len(), 2);+ assert_eq!(result[0], lit(1));+ assert_eq!(result[1], lit(10));++ // Test with reordering and different cases+ let args2 = vec![lit(20), lit(5)];+ let arg_names2 = vec![+ Some(ArgumentName {+ value: "Length".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "StartPos".to_string(),+ is_quoted: false,+ }),+ ];++ let result2 =+ resolve_function_arguments(¶m_names, args2, arg_names2).unwrap();+ assert_eq!(result2.len(), 2);+ assert_eq!(result2[0], lit(5)); // startpos+ assert_eq!(result2[1], lit(20)); // length+ }++ #[test]+ fn test_quoted_parameter_case_sensitive() {+ // Parameter names in function signature (lowercase)+ let param_names = vec!["str".to_string(), "start_pos".to_string()];++ // Quoted identifiers with wrong case should fail+ let args = vec![lit("hello"), lit(1)];+ let arg_names = vec![+ Some(ArgumentName {+ value: "STR".to_string(),+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "start_pos".to_string(),+ is_quoted: true,+ }),+ ];++ let result = resolve_function_arguments(¶m_names, args, arg_names);+ assert!(result.is_err());+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("Unknown parameter")+ );++ // Quoted identifiers with correct case should succeed+ let args2 = vec![lit("hello"), lit(1)];+ let arg_names2 = vec![+ Some(ArgumentName {+ value: "str".to_string(),+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "start_pos".to_string(),+ is_quoted: true,+ }),+ ];++ let result2 =+ resolve_function_arguments(¶m_names, args2, arg_names2).unwrap();+ assert_eq!(result2.len(), 2);+ assert_eq!(result2[0], lit("hello"));+ assert_eq!(result2[1], lit(1));}#[test]@@ -188,9 +309,18 @@ mod tests {// Call with: func(c => 3.0, a => 1, b => "hello")let args = vec![lit(3.0), lit(1), lit("hello")];let arg_names = vec![- Some("c".to_string()),- Some("a".to_string()),- Some("b".to_string()),+ Some(ArgumentName {+ value: "c".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "a".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "b".to_string(),+ is_quoted: false,+ }),];let result = resolve_function_arguments(¶m_names, args, arg_names).unwrap();@@ -208,7 +338,17 @@ mod tests {// Call with: func(1, c => 3.0, b => "hello")let args = vec![lit(1), lit(3.0), lit("hello")];- let arg_names = vec![None, Some("c".to_string()), Some("b".to_string())];+ let arg_names = vec![+ None,+ Some(ArgumentName {+ value: "c".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "b".to_string(),+ is_quoted: false,+ }),+ ];let result = resolve_function_arguments(¶m_names, args, arg_names).unwrap();@@ -225,7 +365,13 @@ mod tests {// Call with: func(a => 1, "hello") - ERRORlet args = vec![lit(1), lit("hello")];- let arg_names = vec![Some("a".to_string()), None];+ let arg_names = vec![+ Some(ArgumentName {+ value: "a".to_string(),+ is_quoted: false,+ }),+ None,+ ];let result = resolve_function_arguments(¶m_names, args, arg_names);assert!(result.is_err());@@ -243,7 +389,16 @@ mod tests {// Call with: func(x => 1, b => "hello") - ERRORlet args = vec![lit(1), lit("hello")];- let arg_names = vec![Some("x".to_string()), Some("b".to_string())];+ let arg_names = vec![+ Some(ArgumentName {+ value: "x".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "b".to_string(),+ is_quoted: false,+ }),+ ];let result = resolve_function_arguments(¶m_names, args, arg_names);assert!(result.is_err());@@ -261,7 +416,16 @@ mod tests {// Call with: func(a => 1, a => 2) - ERRORlet args = vec![lit(1), lit(2)];- let arg_names = vec![Some("a".to_string()), Some("a".to_string())];+ let arg_names = vec![+ Some(ArgumentName {+ value: "a".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "a".to_string(),+ is_quoted: false,+ }),+ ];let result = resolve_function_arguments(¶m_names, args, arg_names);assert!(result.is_err());@@ -279,7 +443,16 @@ mod tests {// Call with: func(a => 1, c => 3.0) - missing 'b'let args = vec![lit(1), lit(3.0)];- let arg_names = vec![Some("a".to_string()), Some("c".to_string())];+ let arg_names = vec![+ Some(ArgumentName {+ value: "a".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "c".to_string(),+ is_quoted: false,+ }),+ ];let result = resolve_function_arguments(¶m_names, args, arg_names);assert!(result.is_err());@@ -290,4 +463,212 @@ mod tests {.contains("Missing required parameter"));}++ #[test]+ fn test_mixed_case_signature_unquoted_matching() {+ // Test with mixed-case signature parameters (lowercase, camelCase, UPPERCASE)+ // This proves case-insensitive matching works for unquoted identifiers+ let param_names = vec![+ "prefix".to_string(), // lowercase+ "startPos".to_string(), // camelCase+ "LENGTH".to_string(), // UPPERCASE+ ];++ // Test 1: All lowercase unquoted arguments should match+ let args1 = vec![lit("a"), lit(1), lit(5)];+ let arg_names1 = vec![+ Some(ArgumentName {+ value: "prefix".to_string(),+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "startpos".to_string(), // lowercase version of startPos+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "length".to_string(), // lowercase version of LENGTH+ is_quoted: false,+ }),+ ];++ let result1 =+ resolve_function_arguments(¶m_names, args1, arg_names1).unwrap();+ assert_eq!(result1.len(), 3);+ assert_eq!(result1[0], lit("a"));+ assert_eq!(result1[1], lit(1));+ assert_eq!(result1[2], lit(5));++ // Test 2: All uppercase unquoted arguments should match+ let args2 = vec![lit("b"), lit(2), lit(10)];+ let arg_names2 = vec![+ Some(ArgumentName {+ value: "PREFIX".to_string(), // uppercase version of prefix+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "STARTPOS".to_string(), // uppercase version of startPos+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "LENGTH".to_string(), // matches UPPERCASE+ is_quoted: false,+ }),+ ];++ let result2 =+ resolve_function_arguments(¶m_names, args2, arg_names2).unwrap();+ assert_eq!(result2.len(), 3);+ assert_eq!(result2[0], lit("b"));+ assert_eq!(result2[1], lit(2));+ assert_eq!(result2[2], lit(10));++ // Test 3: Mixed case unquoted arguments should match+ let args3 = vec![lit("c"), lit(3), lit(15)];+ let arg_names3 = vec![+ Some(ArgumentName {+ value: "Prefix".to_string(), // Title case+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "StartPos".to_string(), // matches camelCase+ is_quoted: false,+ }),+ Some(ArgumentName {+ value: "Length".to_string(), // Title case+ is_quoted: false,+ }),+ ];++ let result3 =+ resolve_function_arguments(¶m_names, args3, arg_names3).unwrap();+ assert_eq!(result3.len(), 3);+ assert_eq!(result3[0], lit("c"));+ assert_eq!(result3[1], lit(3));+ assert_eq!(result3[2], lit(15));+ }++ #[test]+ fn test_mixed_case_signature_quoted_matching() {+ // Test that quoted identifiers require exact case match with signature+ let param_names = vec![+ "prefix".to_string(), // lowercase+ "startPos".to_string(), // camelCase+ "LENGTH".to_string(), // UPPERCASE+ ];++ // Test 1: Quoted with wrong case should fail for "prefix"+ let args_wrong_prefix = vec![lit("a"), lit(1), lit(5)];+ let arg_names_wrong_prefix = vec![+ Some(ArgumentName {+ value: "PREFIX".to_string(), // Wrong case+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "startPos".to_string(),+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "LENGTH".to_string(),+ is_quoted: true,+ }),+ ];++ let result = resolve_function_arguments(+ ¶m_names,+ args_wrong_prefix,+ arg_names_wrong_prefix,+ );+ assert!(result.is_err());+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("Unknown parameter")+ );++ // Test 2: Quoted with wrong case should fail for "startPos"+ let args_wrong_startpos = vec![lit("a"), lit(1), lit(5)];+ let arg_names_wrong_startpos = vec![+ Some(ArgumentName {+ value: "prefix".to_string(),+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "STARTPOS".to_string(), // Wrong case+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "LENGTH".to_string(),+ is_quoted: true,+ }),+ ];++ let result2 = resolve_function_arguments(+ ¶m_names,+ args_wrong_startpos,+ arg_names_wrong_startpos,+ );+ assert!(result2.is_err());+ assert!(+ result2+ .unwrap_err()+ .to_string()+ .contains("Unknown parameter")+ );++ // Test 3: Quoted with wrong case should fail for "LENGTH"+ let args_wrong_length = vec![lit("a"), lit(1), lit(5)];+ let arg_names_wrong_length = vec![+ Some(ArgumentName {+ value: "prefix".to_string(),+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "startPos".to_string(),+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "length".to_string(), // Wrong case+ is_quoted: true,+ }),+ ];++ let result3 = resolve_function_arguments(+ ¶m_names,+ args_wrong_length,+ arg_names_wrong_length,+ );+ assert!(result3.is_err());+ assert!(+ result3+ .unwrap_err()+ .to_string()+ .contains("Unknown parameter")+ );++ // Test 4: Quoted with exact case should succeed+ let args_correct = vec![lit("a"), lit(1), lit(5)];+ let arg_names_correct = vec![+ Some(ArgumentName {+ value: "prefix".to_string(), // Exact match+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "startPos".to_string(), // Exact match+ is_quoted: true,+ }),+ Some(ArgumentName {+ value: "LENGTH".to_string(), // Exact match+ is_quoted: true,+ }),+ ];++ let result4 =+ resolve_function_arguments(¶m_names, args_correct, arg_names_correct)+ .unwrap();+ assert_eq!(result4.len(), 3);+ assert_eq!(result4[0], lit("a"));+ assert_eq!(result4[1], lit(1));+ assert_eq!(result4[2], lit(5));+ }}diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rsindex b29531fdea..641f3bb8dc 100644--- a/datafusion/sql/src/expr/function.rs+++ b/datafusion/sql/src/expr/function.rs@@ -23,7 +23,9 @@ use datafusion_common::{internal_err, not_impl_err, plan_datafusion_err, plan_err,};use datafusion_expr::{- Expr, ExprSchemable, SortExpr, WindowFrame, WindowFunctionDefinition, expr,+ Expr, ExprSchemable, SortExpr, WindowFrame, WindowFunctionDefinition,+ arguments::ArgumentName,+ expr,expr::{NullTreatment, ScalarFunction, Unnest, WildcardOptions, WindowFunction},planner::{PlannerResult, RawAggregateExpr, RawWindowExpr},};@@ -217,7 +219,7 @@ impl FunctionArgs {}// Helper type for extracting WITHIN GROUP ordering and prepended args-type WithinGroupExtraction = (Vec<SortExpr>, Vec<Expr>, Vec<Option<String>>);+type WithinGroupExtraction = (Vec<SortExpr>, Vec<Expr>, Vec<Option<ArgumentName>>);impl<S: ContextProvider> SqlToRel<'_, S> {pub(super) fn sql_function_to_expr(@@ -745,7 +747,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {sql: FunctionArg,schema: &DFSchema,planner_context: &mut PlannerContext,- ) -> Result<(Expr, Option<String>)> {+ ) -> Result<(Expr, Option<ArgumentName>)> {match sql {FunctionArg::Named {name,@@ -753,7 +755,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {operator: _,} => {let expr = self.sql_expr_to_logical_expr(arg, schema, planner_context)?;- let arg_name = crate::utils::normalize_ident(name);+ let arg_name = ArgumentName {+ value: name.value,+ is_quoted: name.quote_style.is_some(),+ };Ok((expr, Some(arg_name)))}FunctionArg::Named {@@ -766,7 +771,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {qualifier: None,options: Box::new(WildcardOptions::default()),};- let arg_name = crate::utils::normalize_ident(name);+ let arg_name = ArgumentName {+ value: name.value,+ is_quoted: name.quote_style.is_some(),+ };Ok((expr, Some(arg_name)))}FunctionArg::Unnamed(FunctionArgExpr::Expr(arg)) => {@@ -803,7 +811,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {operator: _,} => {let expr = self.sql_expr_to_logical_expr(arg, schema, planner_context)?;- let arg_name = crate::utils::normalize_ident(name);+ let arg_name = ArgumentName {+ value: name.value,+ is_quoted: name.quote_style.is_some(),+ };Ok((expr, Some(arg_name)))}FunctionArg::ExprNamed {@@ -816,7 +827,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {qualifier: None,options: Box::new(WildcardOptions::default()),};- let arg_name = crate::utils::normalize_ident(name);+ let arg_name = ArgumentName {+ value: name.value,+ is_quoted: name.quote_style.is_some(),+ };Ok((expr, Some(arg_name)))}_ => not_impl_err!("Unsupported qualified wildcard argument: {sql:?}"),@@ -839,8 +853,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {args: Vec<FunctionArg>,schema: &DFSchema,planner_context: &mut PlannerContext,- ) -> Result<(Vec<Expr>, Vec<Option<String>>)> {- let results: Result<Vec<(Expr, Option<String>)>> = args+ ) -> Result<(Vec<Expr>, Vec<Option<ArgumentName>>)> {+ let results: Result<Vec<(Expr, Option<ArgumentName>)>> = args.into_iter().map(|a| {self.sql_fn_arg_to_logical_expr_with_name(a, schema, planner_context)@@ -848,7 +862,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {.collect();let pairs = results?;- let (exprs, names): (Vec<Expr>, Vec<Option<String>>) = pairs.into_iter().unzip();+ let (exprs, names): (Vec<Expr>, Vec<Option<ArgumentName>>) =+ pairs.into_iter().unzip();Ok((exprs, names))}@@ -856,7 +871,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {&self,within_group: Vec<OrderByExpr>,mut args: Vec<Expr>,- mut arg_names: Vec<Option<String>>,+ mut arg_names: Vec<Option<ArgumentName>>,schema: &DFSchema,planner_context: &mut PlannerContext,) -> Result<WithinGroupExtraction> {diff --git a/datafusion/sqllogictest/test_files/named_arguments.slt b/datafusion/sqllogictest/test_files/named_arguments.sltindex a16de826a2..07b6cc6a79 100644--- a/datafusion/sqllogictest/test_files/named_arguments.slt+++ b/datafusion/sqllogictest/test_files/named_arguments.slt@@ -79,7 +79,8 @@ SELECT substr(Str => 'hello world', Start_Pos => 7);----world-# Error: case-sensitive quoted parameter names don't match+# Error: quoted identifiers are case-sensitive per SQL standards+# "STR" does not match parameter "str" (wrong case)query error DataFusion error: Error during planning: Unknown parameter name 'STR'SELECT substr("STR" => 'hello world', "start_pos" => 7);