Apache DataFusion SQL Query Engine
fix: preserve UPPER and LOWER input nullability (#25381)
Artem Osipovb300cea226af
diff --git a/datafusion/functions/src/string/lower.rs b/datafusion/functions/src/string/lower.rsindex 88f2c800e9..18e718526f 100644--- a/datafusion/functions/src/string/lower.rs+++ b/datafusion/functions/src/string/lower.rs@@ -15,14 +15,14 @@// specific language governing permissions and limitations// under the License.-use arrow::datatypes::DataType;+use arrow::datatypes::{DataType, Field, FieldRef};use crate::string::common::to_lower;use datafusion_common::Result;use datafusion_common::types::logical_string;use datafusion_expr::{- Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,- ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ReturnFieldArgs,+ ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,};use datafusion_macros::user_doc;@@ -80,6 +80,14 @@ impl ScalarUDFImpl for LowerFunc {Ok(arg_types[0].clone())}+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let input = &args.arg_fields[0];+ Ok(+ Field::new(self.name(), input.data_type().clone(), input.is_nullable())+ .into(),+ )+ }+fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {to_lower(&args.args, "lower")}@@ -97,6 +105,21 @@ mod tests {use datafusion_common::config::ConfigOptions;use std::sync::Arc;+ #[test]+ fn preserves_input_nullability() -> Result<()> {+ let func = LowerFunc::new();+ for nullable in [false, true] {+ let input = Field::new("input", DataType::Utf8, nullable);+ let result = func.return_field_from_args(ReturnFieldArgs {+ arg_fields: &[input.into()],+ scalar_arguments: &[None],+ })?;+ assert_eq!(result.data_type(), &DataType::Utf8);+ assert_eq!(result.is_nullable(), nullable);+ }+ Ok(())+ }+fn invoke_lower(input: ArrayRef) -> Result<ArrayRef> {let func = LowerFunc::new();let data_type = input.data_type().clone();diff --git a/datafusion/functions/src/string/upper.rs b/datafusion/functions/src/string/upper.rsindex 789ab2c046..c7aa430d97 100644--- a/datafusion/functions/src/string/upper.rs+++ b/datafusion/functions/src/string/upper.rs@@ -16,12 +16,12 @@// under the License.use crate::string::common::to_upper;-use arrow::datatypes::DataType;+use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::Result;use datafusion_common::types::logical_string;use datafusion_expr::{- Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,- ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, Documentation, EncodingPreservation, ReturnFieldArgs,+ ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,};use datafusion_macros::user_doc;@@ -79,6 +79,14 @@ impl ScalarUDFImpl for UpperFunc {Ok(arg_types[0].clone())}+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let input = &args.arg_fields[0];+ Ok(+ Field::new(self.name(), input.data_type().clone(), input.is_nullable())+ .into(),+ )+ }+fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {to_upper(&args.args, "upper")}@@ -96,6 +104,21 @@ mod tests {use datafusion_common::config::ConfigOptions;use std::sync::Arc;+ #[test]+ fn preserves_input_nullability() -> Result<()> {+ let func = UpperFunc::new();+ for nullable in [false, true] {+ let input = Field::new("input", DataType::Utf8, nullable);+ let result = func.return_field_from_args(ReturnFieldArgs {+ arg_fields: &[input.into()],+ scalar_arguments: &[None],+ })?;+ assert_eq!(result.data_type(), &DataType::Utf8);+ assert_eq!(result.is_nullable(), nullable);+ }+ Ok(())+ }+fn invoke_upper(input: ArrayRef) -> Result<ArrayRef> {let func = UpperFunc::new();let data_type = input.data_type().clone();diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.sltindex 4baee8ce6f..fbabbb07fe 100644--- a/datafusion/sqllogictest/test_files/functions.slt+++ b/datafusion/sqllogictest/test_files/functions.slt@@ -486,6 +486,30 @@ BAR Dictionary(Int32, Utf8)statement okDROP TABLE upper_dictionary_test+# upper and lower preserve input nullability in the planned schema+query TTT+DESCRIBE SELECT upper(a) AS ua, lower(a) AS la, upper(b) AS ub+FROM (VALUES ('Ab', CAST(NULL AS VARCHAR))) AS t(a, b)+----+ua Utf8 NO+la Utf8 NO+ub Utf8View YES++# An outer join widens a non-nullable input before it reaches upper+query TTT+DESCRIBE SELECT upper(t.a) AS ua+FROM (SELECT 1 AS k) x+LEFT JOIN (VALUES ('Ab')) AS t(a) ON false+----+ua Utf8 YES++query T+SELECT upper(t.a)+FROM (SELECT 1 AS k) x+LEFT JOIN (VALUES ('Ab')) AS t(a) ON false+----+NULL+query TSELECT btrim(' foo ')----