Skip to content

Apache DataFusion SQL Query Engine

feat: keep extension types of CREATE FUNCTION arguments and return type (#27102)

Yury Syrovetskye9d45f320a5d
diff --git a/datafusion-examples/examples/builtin_functions/function_factory.rs b/datafusion-examples/examples/builtin_functions/function_factory.rs
index 6aa9994078..b5633b43fb 100644
--- a/datafusion-examples/examples/builtin_functions/function_factory.rs
+++ b/datafusion-examples/examples/builtin_functions/function_factory.rs
@@ -190,14 +190,16 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {
.function_body
.expect("Expression has to be defined!"),
return_type: definition
- .return_type
- .expect("Return type has to be defined!"),
+ .return_field
+ .expect("Return type has to be defined!")
+ .data_type()
+ .clone(),
signature: Signature::exact(
definition
.args
.unwrap_or_default()
.into_iter()
- .map(|a| a.data_type)
+ .map(|a| a.field.data_type().clone())
.collect(),
definition.params.behavior.unwrap_or(Volatility::Volatile),
),
diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
index c5fd2e1de1..d391cb5018 100644
--- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
+++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
@@ -13,13 +13,14 @@ use arrow::array::{
use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch};
use arrow::compute::kernels::numeric::add;
use arrow::datatypes::{DataType, Field, Metadata, Schema};
-use arrow_schema::extension::{Bool8, CanonicalExtensionType, ExtensionType};
+use arrow_schema::extension::{Bool8, CanonicalExtensionType, ExtensionType, Uuid};
use arrow_schema::{ArrowError, FieldRef, SchemaRef};
use datafusion::common::test_util::batches_to_string;
use datafusion::execution::context::{FunctionFactory, RegisterFunction, SessionState};
use datafusion::prelude::*;
use datafusion::{execution::registry::FunctionRegistry, test_util};
use datafusion_common::cast::{as_float64_array, as_int32_array};
+use datafusion_common::datatype::DataTypeExt;
use datafusion_common::metadata::FieldMetadata;
use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::utils::take_function_args;
@@ -33,7 +34,7 @@ use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
use datafusion_expr::{
Accumulator, ColumnarValue, CreateFunction, CreateFunctionBody, LogicalPlanBuilder,
OperateFunctionArg, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
- Signature, TypeSignatureClass, Volatility, lit_with_metadata,
+ Signature, TypeSignatureClass, Volatility, expr::Placeholder, lit_with_metadata,
};
use datafusion_expr_common::signature::Coercion;
use datafusion_expr_common::signature::TypeSignature;
@@ -1016,7 +1017,10 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {
type_signatures.push(TypeSignature::Nullary)
} else {
type_signatures.push(TypeSignature::Exact(
- args.iter().take(n).map(|a| a.data_type.clone()).collect(),
+ args.iter()
+ .take(n)
+ .map(|a| a.field.data_type().clone())
+ .collect(),
))
}
}
@@ -1026,7 +1030,7 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {
)
}
None => Signature::exact(
- args.iter().map(|a| a.data_type.clone()).collect(),
+ args.iter().map(|a| a.field.data_type().clone()).collect(),
definition.params.behavior.unwrap_or(Volatility::Volatile),
),
};
@@ -1037,8 +1041,10 @@ impl TryFrom<CreateFunction> for ScalarFunctionWrapper {
.function_body
.expect("Expression has to be defined!"),
return_type: definition
- .return_type
- .expect("Return type has to be defined!"),
+ .return_field
+ .expect("Return type has to be defined!")
+ .data_type()
+ .clone(),
signature,
defaults,
})
@@ -1337,10 +1343,10 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
quote_style: None,
span: Span::empty(),
}),
- data_type: DataType::Utf8View,
+ field: Arc::new(Field::new("name", DataType::Utf8View, true)),
default_expr: None,
}]),
- return_type: Some(DataType::Int32),
+ return_field: Some(DataType::Int32.into_nullable_field_ref()),
params: CreateFunctionBody {
language: Some(Ident {
value: "plrust".into(),
@@ -1358,6 +1364,39 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
Ok(())
}
+#[tokio::test]
+async fn create_function_keeps_extension_types() -> Result<()> {
+ let function_factory = Arc::new(RecordingFunctionFactory::new());
+ let ctx = SessionContext::new().with_function_factory(function_factory.clone());
+
+ ctx.sql("CREATE FUNCTION same_uuid(id UUID) RETURNS UUID RETURN $1")
+ .await?;
+
+ let calls = function_factory.calls();
+ assert_eq!(calls.len(), 1);
+ let call = &calls[0];
+ let uuid_field = |name| {
+ Arc::new(
+ Field::new(name, DataType::FixedSizeBinary(16), true)
+ .with_extension_type(Uuid),
+ )
+ };
+ let args = call.args.as_ref().expect("arguments");
+ assert_eq!(args.len(), 1);
+ assert_eq!(args[0].field, uuid_field("id"));
+ assert_eq!(call.return_field, Some(uuid_field("")));
+ // The placeholder in the body has the type of the argument
+ assert_eq!(
+ call.params.function_body,
+ Some(Expr::Placeholder(Placeholder::new_with_field(
+ "$1".to_string(),
+ Some(uuid_field("id")),
+ )))
+ );
+
+ Ok(())
+}
+
#[derive(Debug)]
struct MyRegexUdf {
signature: Signature,
diff --git a/datafusion/expr/src/logical_plan/ddl.rs b/datafusion/expr/src/logical_plan/ddl.rs
index 81d6fac189..b4b0847ef6 100644
--- a/datafusion/expr/src/logical_plan/ddl.rs
+++ b/datafusion/expr/src/logical_plan/ddl.rs
@@ -14,7 +14,7 @@ use std::{
use crate::expr::{Exists, InSubquery, SetComparison, Sort};
#[cfg(not(feature = "sql"))]
use crate::sql::Ident;
-use arrow::datatypes::DataType;
+use arrow::datatypes::FieldRef;
use datafusion_common::tree_node::{
Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,
};
@@ -900,7 +900,8 @@ pub struct CreateFunction {
pub temporary: bool,
pub name: String,
pub args: Option<Vec<OperateFunctionArg>>,
- pub return_type: Option<DataType>,
+ /// The return type, which may carry metadata such as an extension type
+ pub return_field: Option<FieldRef>,
pub params: CreateFunctionBody,
/// Dummy schema
pub schema: DFSchemaRef,
@@ -918,7 +919,7 @@ impl CreateFunction {
temporary: bool,
name: String,
args: Option<Vec<OperateFunctionArg>>,
- return_type: Option<DataType>,
+ return_field: Option<FieldRef>,
params: CreateFunctionBody,
) -> Result<Self> {
let arg_count = args.as_ref().map_or(0, |declared| declared.len());
@@ -937,7 +938,7 @@ impl CreateFunction {
temporary,
name,
args,
- return_type,
+ return_field,
params,
schema: DFSchemaRef::new(DFSchema::empty()),
})
@@ -1004,7 +1005,7 @@ impl PartialOrd for CreateFunction {
pub temporary: &'a bool,
pub name: &'a String,
pub args: &'a Option<Vec<OperateFunctionArg>>,
- pub return_type: &'a Option<DataType>,
+ pub return_field: &'a Option<FieldRef>,
pub params: &'a CreateFunctionBody,
}
let comparable_self = ComparableCreateFunction {
@@ -1012,7 +1013,7 @@ impl PartialOrd for CreateFunction {
temporary: &self.temporary,
name: &self.name,
args: &self.args,
- return_type: &self.return_type,
+ return_field: &self.return_field,
params: &self.params,
};
let comparable_other = ComparableCreateFunction {
@@ -1020,7 +1021,7 @@ impl PartialOrd for CreateFunction {
temporary: &other.temporary,
name: &other.name,
args: &other.args,
- return_type: &other.return_type,
+ return_field: &other.return_field,
params: &other.params,
};
comparable_self
@@ -1038,7 +1039,8 @@ pub struct OperateFunctionArg {
// TODO: figure out how to support mode
// pub mode: Option<ArgMode>,
pub name: Option<Ident>,
- pub data_type: DataType,
+ /// The argument type, which may carry metadata such as an extension type
+ pub field: FieldRef,
pub default_expr: Option<Expr>,
}
@@ -1172,6 +1174,7 @@ mod test {
LogicalPlanBuilder, Subquery,
};
use arrow::datatypes::DataType;
+ use datafusion_common::datatype::DataTypeExt;
use datafusion_common::{DFSchema, DFSchemaRef, TableReference};
use std::{cmp::Ordering, collections::HashMap, sync::Arc};
@@ -1184,10 +1187,10 @@ mod test {
"f".to_string(),
Some(vec![OperateFunctionArg {
name: None,
- data_type: DataType::Int64,
+ field: DataType::Int64.into_nullable_field_ref(),
default_expr,
}]),
- Some(DataType::Int64),
+ Some(DataType::Int64.into_nullable_field_ref()),
CreateFunctionBody {
language: None,
behavior: None,
diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index e3701bc1a3..967d42a428 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -14,7 +14,8 @@ use crate::parser::{
use crate::planner::{ContextProvider, PlannerContext, SqlToRel};
use crate::utils::normalize_ident;
-use arrow::datatypes::{Field, FieldRef, Fields, Metadata};
+use arrow::datatypes::{FieldRef, Fields, Metadata};
+use datafusion_common::datatype::FieldExt;
use datafusion_common::error::_plan_err;
use datafusion_common::format::ExplainStatementOptions;
use datafusion_common::parsers::CompressionTypeVariant;
@@ -1385,8 +1386,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
let function_args = function_args
.into_iter()
.map(|arg| {
- let data_type =
- self.convert_data_type_to_field(&arg.data_type)?;
+ // Name the field after the argument, so that it can also
+ // be used as the type of the placeholder in the body
+ let field = self
+ .convert_data_type_to_field(&arg.data_type)?
+ .renamed(
+ arg.name.as_ref().map_or("", |name| &name.value),
+ );
let default_expr = match arg.default_expr {
Some(expr) => Some(self.sql_to_expr(
@@ -1399,7 +1405,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
Ok(OperateFunctionArg {
name: arg.name,
default_expr,
- data_type: data_type.data_type().clone(),
+ field,
})
})
.collect::<Result<Vec<OperateFunctionArg>>>();
@@ -1432,15 +1438,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// Convert resulting expression to data fusion expression
//
let arg_types = args.as_ref().map(|arg| {
- arg.iter()
- .map(|t| {
- let name = match t.name.clone() {
- Some(name) => name.value,
- None => "".to_string(),
- };
- Arc::new(Field::new(name, t.data_type.clone(), true))
- })
- .collect::<Vec<_>>()
+ arg.iter().map(|t| Arc::clone(&t.field)).collect::<Vec<_>>()
});
// Validate parameter style
if let Some(ref fields) = arg_types {
@@ -1502,7 +1500,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
temporary,
name,
args,
- return_type.map(|f| f.data_type().clone()),
+ return_type,
params,
)?));
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 849f3237d8..07714dcd55 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1593,3 +1593,31 @@ fn is_nullable(&self) -> bool {
```
See [issue #18882](https://github.com/apache/datafusion/issues/18882) for details.
+
+### `CreateFunction` argument and return types are `FieldRef`s
+
+`OperateFunctionArg::data_type` is replaced by `OperateFunctionArg::field`, and
+`CreateFunction::return_type` by `CreateFunction::return_field`. Both are now
+`FieldRef`s instead of `DataType`s, so `CREATE FUNCTION` can declare argument
+and return types that need field metadata, such as extension types
+(`CREATE FUNCTION f(id UUID) RETURNS UUID ...`). The argument field is named
+after the argument, or `""` if the argument has no name. The return field has
+the name `""`.
+
+**Migration guide:**
+
+```rust,ignore
+// Before
+let arg_types: Vec<DataType> = args.iter().map(|a| a.data_type.clone()).collect();
+let return_type: Option<DataType> = definition.return_type;
+
+// After
+let arg_types: Vec<DataType> = args.iter().map(|a| a.field.data_type().clone()).collect();
+let return_type: Option<DataType> = definition
+ .return_field
+ .map(|field| field.data_type().clone());
+```
+
+`CreateFunction::try_new` takes `Option<FieldRef>` for the return type.
+
+See [issue #18235](https://github.com/apache/datafusion/issues/18235) for details.