Skip to content

Apache DataFusion SQL Query Engine

Use return_field_from_args in information schema and date_trunc (#20079)

Andrea Bozzo1a0c2e05c4b4
diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs
index 52bfeca3d4..ea93dc21a3 100644
--- a/datafusion/catalog/src/information_schema.rs
+++ b/datafusion/catalog/src/information_schema.rs
@@ -24,7 +24,7 @@ use crate::{CatalogProviderList, SchemaProvider, TableProvider};
use arrow::array::builder::{BooleanBuilder, UInt8Builder};
use arrow::{
array::{StringBuilder, UInt64Builder},
- datatypes::{DataType, Field, Schema, SchemaRef},
+ datatypes::{DataType, Field, FieldRef, Schema, SchemaRef},
record_batch::RecordBatch,
};
use async_trait::async_trait;
@@ -34,7 +34,10 @@ use datafusion_common::error::Result;
use datafusion_common::types::NativeType;
use datafusion_execution::TaskContext;
use datafusion_execution::runtime_env::RuntimeEnv;
-use datafusion_expr::{AggregateUDF, ScalarUDF, Signature, TypeSignature, WindowUDF};
+use datafusion_expr::function::WindowUDFFieldArgs;
+use datafusion_expr::{
+ AggregateUDF, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature, WindowUDF,
+};
use datafusion_expr::{TableType, Volatility};
use datafusion_physical_plan::SendableRecordBatchStream;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
@@ -421,10 +424,24 @@ fn get_udf_args_and_return_types(
Ok(arg_types
.into_iter()
.map(|arg_types| {
- // only handle the function which implemented [`ScalarUDFImpl::return_type`] method
+ let arg_fields: Vec<FieldRef> = arg_types
+ .iter()
+ .enumerate()
+ .map(|(i, t)| {
+ Arc::new(Field::new(format!("arg_{i}"), t.clone(), true))
+ })
+ .collect();
+ let scalar_arguments = vec![None; arg_fields.len()];
let return_type = udf
- .return_type(&arg_types)
- .map(|t| remove_native_type_prefix(&NativeType::from(t)))
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &arg_fields,
+ scalar_arguments: &scalar_arguments,
+ })
+ .map(|f| {
+ remove_native_type_prefix(&NativeType::from(
+ f.data_type().clone(),
+ ))
+ })
.ok();
let arg_types = arg_types
.into_iter()
@@ -447,11 +464,21 @@ fn get_udaf_args_and_return_types(
Ok(arg_types
.into_iter()
.map(|arg_types| {
- // only handle the function which implemented [`ScalarUDFImpl::return_type`] method
+ let arg_fields: Vec<FieldRef> = arg_types
+ .iter()
+ .enumerate()
+ .map(|(i, t)| {
+ Arc::new(Field::new(format!("arg_{i}"), t.clone(), true))
+ })
+ .collect();
let return_type = udaf
- .return_type(&arg_types)
- .ok()
- .map(|t| remove_native_type_prefix(&NativeType::from(t)));
+ .return_field(&arg_fields)
+ .map(|f| {
+ remove_native_type_prefix(&NativeType::from(
+ f.data_type().clone(),
+ ))
+ })
+ .ok();
let arg_types = arg_types
.into_iter()
.map(|t| remove_native_type_prefix(&NativeType::from(t)))
@@ -473,12 +500,26 @@ fn get_udwf_args_and_return_types(
Ok(arg_types
.into_iter()
.map(|arg_types| {
- // only handle the function which implemented [`ScalarUDFImpl::return_type`] method
+ let arg_fields: Vec<FieldRef> = arg_types
+ .iter()
+ .enumerate()
+ .map(|(i, t)| {
+ Arc::new(Field::new(format!("arg_{i}"), t.clone(), true))
+ })
+ .collect();
+ let return_type = udwf
+ .field(WindowUDFFieldArgs::new(&arg_fields, udwf.name()))
+ .map(|f| {
+ remove_native_type_prefix(&NativeType::from(
+ f.data_type().clone(),
+ ))
+ })
+ .ok();
let arg_types = arg_types
.into_iter()
.map(|t| remove_native_type_prefix(&NativeType::from(t)))
.collect::<Vec<_>>();
- (arg_types, None)
+ (arg_types, return_type)
})
.collect::<BTreeSet<_>>())
}
diff --git a/datafusion/functions/benches/date_trunc.rs b/datafusion/functions/benches/date_trunc.rs
index f5c8ceb5fe..664221f7ba 100644
--- a/datafusion/functions/benches/date_trunc.rs
+++ b/datafusion/functions/benches/date_trunc.rs
@@ -25,7 +25,7 @@ use arrow::datatypes::Field;
use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs};
use datafusion_functions::datetime::date_trunc;
use rand::Rng;
use rand::rngs::ThreadRng;
@@ -57,10 +57,13 @@ fn criterion_benchmark(c: &mut Criterion) {
})
.collect::<Vec<_>>();
- let return_type = udf
- .return_type(&args.iter().map(|arg| arg.data_type()).collect::<Vec<_>>())
+ let scalar_arguments = vec![None; arg_fields.len()];
+ let return_field = udf
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &arg_fields,
+ scalar_arguments: &scalar_arguments,
+ })
.unwrap();
- let return_field = Arc::new(Field::new("f", return_type, true));
let config_options = Arc::new(ConfigOptions::default());
b.iter(|| {
diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs
index 951ce7e882..8497e583ba 100644
--- a/datafusion/functions/src/datetime/date_trunc.rs
+++ b/datafusion/functions/src/datetime/date_trunc.rs
@@ -38,7 +38,7 @@ use arrow::datatypes::{Field, FieldRef};
use datafusion_common::cast::as_primitive_array;
use datafusion_common::types::{NativeType, logical_date, logical_string};
use datafusion_common::{
- DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err,
+ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,
};
use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
use datafusion_expr::{
@@ -223,27 +223,21 @@ impl ScalarUDFImpl for DateTruncFunc {
&self.signature
}
- // keep return_type implementation for information schema generation
- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- if arg_types[1].is_null() {
- Ok(Timestamp(Nanosecond, None))
- } else {
- Ok(arg_types[1].clone())
- }
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be called instead")
}
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
- let data_types = args
- .arg_fields
- .iter()
- .map(|f| f.data_type())
- .cloned()
- .collect::<Vec<_>>();
- let return_type = self.return_type(&data_types)?;
+ let field = &args.arg_fields[1];
+ let return_type = if field.data_type().is_null() {
+ Timestamp(Nanosecond, None)
+ } else {
+ field.data_type().clone()
+ };
Ok(Arc::new(Field::new(
self.name(),
return_type,
- args.arg_fields[1].is_nullable(),
+ field.is_nullable(),
)))
}