Apache DataFusion SQL Query Engine
fix: report a non-nullable return field for named_struct() and struct() (#25306)
Adrian Garcia Badaracco32662e750116
diff --git a/datafusion/functions/src/core/named_struct.rs b/datafusion/functions/src/core/named_struct.rsindex e1aa441e0b..0d9e133ada 100644--- a/datafusion/functions/src/core/named_struct.rs+++ b/datafusion/functions/src/core/named_struct.rs@@ -141,10 +141,13 @@ impl ScalarUDFImpl for NamedStructFunc {.map(|(name, data_type)| Ok(Field::new(name, data_type.to_owned(), true))).collect::<Result<Vec<Field>>>()?;+ // The constructed struct row is never NULL: `invoke_with_args` builds+ // the `StructArray` without a null buffer, so the output field is+ // non-nullable and `named_struct(...) IS NOT NULL` folds to `true`.Ok(Field::new(self.name(),DataType::Struct(Fields::from(return_fields)),- true,+ false,).into())}diff --git a/datafusion/functions/src/core/struct.rs b/datafusion/functions/src/core/struct.rsindex 2697cb46b0..164b9d2032 100644--- a/datafusion/functions/src/core/struct.rs+++ b/datafusion/functions/src/core/struct.rs@@ -16,9 +16,11 @@// under the License.use arrow::array::StructArray;-use arrow::datatypes::{DataType, Field};+use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::{Result, exec_err, internal_err};-use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs};+use datafusion_expr::{+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,+};use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};use datafusion_macros::user_doc;use std::sync::Arc;@@ -114,6 +116,18 @@ impl ScalarUDFImpl for StructFunc {Ok(DataType::Struct(fields))}+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let arg_types = args+ .arg_fields+ .iter()+ .map(|f| f.data_type().clone())+ .collect::<Vec<_>>();+ // The constructed struct row is never NULL: `invoke_with_args` builds+ // the `StructArray` without a null buffer, so the output field is+ // non-nullable and `struct(...) IS NOT NULL` folds to `true`.+ Ok(Field::new(self.name(), self.return_type(&arg_types)?, false).into())+ }+fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {let DataType::Struct(fields) = args.return_type() else {return internal_err!("incorrect struct return type");diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex 183bfbd04e..7704b7f214 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -1752,3 +1752,53 @@ limit 2;statement okdrop table list_cast_limit;++# named_struct() and struct() never produce a NULL row, so their return field+# is non-nullable and `IS NOT NULL` on the constructor folds to `true`. A guard+# on a struct built by a view then no longer keeps the whole struct alive, and+# the scan is pruned to the fields that are actually read.+statement ok+create table struct_ctor_null (a int, b int, c int) as values (1, 2, 3), (NULL, 5, 6);++statement ok+create view struct_ctor_view as select named_struct('a', a, 'b', b, 'c', c) as s from struct_ctor_null;++query BB+select named_struct('a', a) is not null, struct(a) is null from struct_ctor_null;+----+true false+true false++query TT+explain select s['b'] from struct_ctor_view where s is not null;+----+logical_plan+01)Projection: __datafusion_extracted_1 AS struct_ctor_view.s[b]+02)--SubqueryAlias: struct_ctor_view+03)----Projection: struct_ctor_null.b AS __datafusion_extracted_1+04)------TableScan: struct_ctor_null projection=[b]+physical_plan+01)ProjectionExec: expr=[b@0 as struct_ctor_view.s[b]]+02)--DataSourceExec: partitions=1, partition_sizes=[1]++query I+select s['b'] from struct_ctor_view where s is not null;+----+2+5++query TT+explain select case when named_struct('a', a) is not null then b end from struct_ctor_null;+----+logical_plan+01)Projection: struct_ctor_null.b AS CASE WHEN named_struct(Utf8("a"),struct_ctor_null.a) IS NOT NULL THEN struct_ctor_null.b END+02)--TableScan: struct_ctor_null projection=[b]+physical_plan+01)ProjectionExec: expr=[b@0 as CASE WHEN named_struct(Utf8("a"),struct_ctor_null.a) IS NOT NULL THEN struct_ctor_null.b END]+02)--DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+drop view struct_ctor_view;++statement ok+drop table struct_ctor_null;