Apache DataFusion SQL Query Engine
feat: keep extension types of CREATE FUNCTION arguments and return type (#27102)
Yury Syrovetskye9d45f320a5d
diff --git a/datafusion-examples/examples/builtin_functions/function_factory.rs b/datafusion-examples/examples/builtin_functions/function_factory.rsindex 6aa9994078..b5633b43fb 100644--- a/datafusion-examples/examples/builtin_functions/function_factory.rs+++ b/datafusion-examples/examples/builtin_functions/function_factory.rs@@ -190,14 +190,16 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {.function_body.expect("Expression has to be defined!"),return_type: definition- .return_type- .expect("Return type has to be defined!"),+ .return_field+ .expect("Return type has to be defined!")+ .data_type()+ .clone(),signature: Signature::exact(definition.args.unwrap_or_default().into_iter()- .map(|a| a.data_type)+ .map(|a| a.field.data_type().clone()).collect(),definition.params.behavior.unwrap_or(Volatility::Volatile),),diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rsindex c5fd2e1de1..d391cb5018 100644--- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs@@ -13,13 +13,14 @@ use arrow::array::{use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch};use arrow::compute::kernels::numeric::add;use arrow::datatypes::{DataType, Field, Metadata, Schema};-use arrow_schema::extension::{Bool8, CanonicalExtensionType, ExtensionType};+use arrow_schema::extension::{Bool8, CanonicalExtensionType, ExtensionType, Uuid};use arrow_schema::{ArrowError, FieldRef, SchemaRef};use datafusion::common::test_util::batches_to_string;use datafusion::execution::context::{FunctionFactory, RegisterFunction, SessionState};use datafusion::prelude::*;use datafusion::{execution::registry::FunctionRegistry, test_util};use datafusion_common::cast::{as_float64_array, as_int32_array};+use datafusion_common::datatype::DataTypeExt;use datafusion_common::metadata::FieldMetadata;use datafusion_common::tree_node::{Transformed, TreeNode};use datafusion_common::utils::take_function_args;@@ -33,7 +34,7 @@ use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};use datafusion_expr::{Accumulator, ColumnarValue, CreateFunction, CreateFunctionBody, LogicalPlanBuilder,OperateFunctionArg, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,- Signature, TypeSignatureClass, Volatility, lit_with_metadata,+ Signature, TypeSignatureClass, Volatility, expr::Placeholder, lit_with_metadata,};use datafusion_expr_common::signature::Coercion;use datafusion_expr_common::signature::TypeSignature;@@ -1016,7 +1017,10 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {type_signatures.push(TypeSignature::Nullary)} else {type_signatures.push(TypeSignature::Exact(- args.iter().take(n).map(|a| a.data_type.clone()).collect(),+ args.iter()+ .take(n)+ .map(|a| a.field.data_type().clone())+ .collect(),))}}@@ -1026,7 +1030,7 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {)}None => Signature::exact(- args.iter().map(|a| a.data_type.clone()).collect(),+ args.iter().map(|a| a.field.data_type().clone()).collect(),definition.params.behavior.unwrap_or(Volatility::Volatile),),};@@ -1037,8 +1041,10 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {.function_body.expect("Expression has to be defined!"),return_type: definition- .return_type- .expect("Return type has to be defined!"),+ .return_field+ .expect("Return type has to be defined!")+ .data_type()+ .clone(),signature,defaults,})@@ -1337,10 +1343,10 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(quote_style: None,span: Span::empty(),}),- data_type: DataType::Utf8View,+ field: Arc::new(Field::new("name", DataType::Utf8View, true)),default_expr: None,}]),- return_type: Some(DataType::Int32),+ return_field: Some(DataType::Int32.into_nullable_field_ref()),params: CreateFunctionBody {language: Some(Ident {value: "plrust".into(),@@ -1358,6 +1364,39 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(Ok(())}+#[tokio::test]+async fn create_function_keeps_extension_types() -> Result<()> {+ let function_factory = Arc::new(RecordingFunctionFactory::new());+ let ctx = SessionContext::new().with_function_factory(function_factory.clone());++ ctx.sql("CREATE FUNCTION same_uuid(id UUID) RETURNS UUID RETURN $1")+ .await?;++ let calls = function_factory.calls();+ assert_eq!(calls.len(), 1);+ let call = &calls[0];+ let uuid_field = |name| {+ Arc::new(+ Field::new(name, DataType::FixedSizeBinary(16), true)+ .with_extension_type(Uuid),+ )+ };+ let args = call.args.as_ref().expect("arguments");+ assert_eq!(args.len(), 1);+ assert_eq!(args[0].field, uuid_field("id"));+ assert_eq!(call.return_field, Some(uuid_field("")));+ // The placeholder in the body has the type of the argument+ assert_eq!(+ call.params.function_body,+ Some(Expr::Placeholder(Placeholder::new_with_field(+ "$1".to_string(),+ Some(uuid_field("id")),+ )))+ );++ Ok(())+}+#[derive(Debug)]struct MyRegexUdf {signature: Signature,diff --git a/datafusion/expr/src/logical_plan/ddl.rs b/datafusion/expr/src/logical_plan/ddl.rsindex 81d6fac189..b4b0847ef6 100644--- a/datafusion/expr/src/logical_plan/ddl.rs+++ b/datafusion/expr/src/logical_plan/ddl.rs@@ -14,7 +14,7 @@ use std::{use crate::expr::{Exists, InSubquery, SetComparison, Sort};#[cfg(not(feature = "sql"))]use crate::sql::Ident;-use arrow::datatypes::DataType;+use arrow::datatypes::FieldRef;use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,};@@ -900,7 +900,8 @@ pub struct CreateFunction {pub temporary: bool,pub name: String,pub args: Option<Vec<OperateFunctionArg>>,- pub return_type: Option<DataType>,+ /// The return type, which may carry metadata such as an extension type+ pub return_field: Option<FieldRef>,pub params: CreateFunctionBody,/// Dummy schemapub schema: DFSchemaRef,@@ -918,7 +919,7 @@ impl CreateFunction {temporary: bool,name: String,args: Option<Vec<OperateFunctionArg>>,- return_type: Option<DataType>,+ return_field: Option<FieldRef>,params: CreateFunctionBody,) -> Result<Self> {let arg_count = args.as_ref().map_or(0, |declared| declared.len());@@ -937,7 +938,7 @@ impl CreateFunction {temporary,name,args,- return_type,+ return_field,params,schema: DFSchemaRef::new(DFSchema::empty()),})@@ -1004,7 +1005,7 @@ impl PartialOrd for CreateFunction {pub temporary: &'a bool,pub name: &'a String,pub args: &'a Option<Vec<OperateFunctionArg>>,- pub return_type: &'a Option<DataType>,+ pub return_field: &'a Option<FieldRef>,pub params: &'a CreateFunctionBody,}let comparable_self = ComparableCreateFunction {@@ -1012,7 +1013,7 @@ impl PartialOrd for CreateFunction {temporary: &self.temporary,name: &self.name,args: &self.args,- return_type: &self.return_type,+ return_field: &self.return_field,params: &self.params,};let comparable_other = ComparableCreateFunction {@@ -1020,7 +1021,7 @@ impl PartialOrd for CreateFunction {temporary: &other.temporary,name: &other.name,args: &other.args,- return_type: &other.return_type,+ return_field: &other.return_field,params: &other.params,};comparable_self@@ -1038,7 +1039,8 @@ pub struct OperateFunctionArg {// TODO: figure out how to support mode// pub mode: Option<ArgMode>,pub name: Option<Ident>,- pub data_type: DataType,+ /// The argument type, which may carry metadata such as an extension type+ pub field: FieldRef,pub default_expr: Option<Expr>,}@@ -1172,6 +1174,7 @@ mod test {LogicalPlanBuilder, Subquery,};use arrow::datatypes::DataType;+ use datafusion_common::datatype::DataTypeExt;use datafusion_common::{DFSchema, DFSchemaRef, TableReference};use std::{cmp::Ordering, collections::HashMap, sync::Arc};@@ -1184,10 +1187,10 @@ mod test {"f".to_string(),Some(vec![OperateFunctionArg {name: None,- data_type: DataType::Int64,+ field: DataType::Int64.into_nullable_field_ref(),default_expr,}]),- Some(DataType::Int64),+ Some(DataType::Int64.into_nullable_field_ref()),CreateFunctionBody {language: None,behavior: None,diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex e3701bc1a3..967d42a428 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -14,7 +14,8 @@ use crate::parser::{use crate::planner::{ContextProvider, PlannerContext, SqlToRel};use crate::utils::normalize_ident;-use arrow::datatypes::{Field, FieldRef, Fields, Metadata};+use arrow::datatypes::{FieldRef, Fields, Metadata};+use datafusion_common::datatype::FieldExt;use datafusion_common::error::_plan_err;use datafusion_common::format::ExplainStatementOptions;use datafusion_common::parsers::CompressionTypeVariant;@@ -1385,8 +1386,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let function_args = function_args.into_iter().map(|arg| {- let data_type =- self.convert_data_type_to_field(&arg.data_type)?;+ // Name the field after the argument, so that it can also+ // be used as the type of the placeholder in the body+ let field = self+ .convert_data_type_to_field(&arg.data_type)?+ .renamed(+ arg.name.as_ref().map_or("", |name| &name.value),+ );let default_expr = match arg.default_expr {Some(expr) => Some(self.sql_to_expr(@@ -1399,7 +1405,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {Ok(OperateFunctionArg {name: arg.name,default_expr,- data_type: data_type.data_type().clone(),+ field,})}).collect::<Result<Vec<OperateFunctionArg>>>();@@ -1432,15 +1438,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// Convert resulting expression to data fusion expression//let arg_types = args.as_ref().map(|arg| {- arg.iter()- .map(|t| {- let name = match t.name.clone() {- Some(name) => name.value,- None => "".to_string(),- };- Arc::new(Field::new(name, t.data_type.clone(), true))- })- .collect::<Vec<_>>()+ arg.iter().map(|t| Arc::clone(&t.field)).collect::<Vec<_>>()});// Validate parameter styleif let Some(ref fields) = arg_types {@@ -1502,7 +1500,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {temporary,name,args,- return_type.map(|f| f.data_type().clone()),+ return_type,params,)?));diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 849f3237d8..07714dcd55 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -1593,3 +1593,31 @@ fn is_nullable(&self) -> bool {```See [issue #18882](https://github.com/apache/datafusion/issues/18882) for details.++### `CreateFunction` argument and return types are `FieldRef`s++`OperateFunctionArg::data_type` is replaced by `OperateFunctionArg::field`, and+`CreateFunction::return_type` by `CreateFunction::return_field`. Both are now+`FieldRef`s instead of `DataType`s, so `CREATE FUNCTION` can declare argument+and return types that need field metadata, such as extension types+(`CREATE FUNCTION f(id UUID) RETURNS UUID ...`). The argument field is named+after the argument, or `""` if the argument has no name. The return field has+the name `""`.++**Migration guide:**++```rust,ignore+// Before+let arg_types: Vec<DataType> = args.iter().map(|a| a.data_type.clone()).collect();+let return_type: Option<DataType> = definition.return_type;++// After+let arg_types: Vec<DataType> = args.iter().map(|a| a.field.data_type().clone()).collect();+let return_type: Option<DataType> = definition+ .return_field+ .map(|field| field.data_type().clone());+```++`CreateFunction::try_new` takes `Option<FieldRef>` for the return type.++See [issue #18235](https://github.com/apache/datafusion/issues/18235) for details.