Skip to content

Apache DataFusion SQL Query Engine

fix: report a non-nullable return field for named_struct() and struct() (#25306)

Adrian Garcia Badaracco32662e750116
diff --git a/datafusion/functions/src/core/named_struct.rs b/datafusion/functions/src/core/named_struct.rs
index e1aa441e0b..0d9e133ada 100644
--- a/datafusion/functions/src/core/named_struct.rs
+++ b/datafusion/functions/src/core/named_struct.rs
@@ -141,10 +141,13 @@ impl ScalarUDFImpl for NamedStructFunc {
.map(|(name, data_type)| Ok(Field::new(name, data_type.to_owned(), true)))
.collect::<Result<Vec<Field>>>()?;
+ // The constructed struct row is never NULL: `invoke_with_args` builds
+ // the `StructArray` without a null buffer, so the output field is
+ // non-nullable and `named_struct(...) IS NOT NULL` folds to `true`.
Ok(Field::new(
self.name(),
DataType::Struct(Fields::from(return_fields)),
- true,
+ false,
)
.into())
}
diff --git a/datafusion/functions/src/core/struct.rs b/datafusion/functions/src/core/struct.rs
index 2697cb46b0..164b9d2032 100644
--- a/datafusion/functions/src/core/struct.rs
+++ b/datafusion/functions/src/core/struct.rs
@@ -16,9 +16,11 @@
// under the License.
use arrow::array::StructArray;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::{Result, exec_err, internal_err};
-use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
+};
use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
use datafusion_macros::user_doc;
use std::sync::Arc;
@@ -114,6 +116,18 @@ impl ScalarUDFImpl for StructFunc {
Ok(DataType::Struct(fields))
}
+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
+ let arg_types = args
+ .arg_fields
+ .iter()
+ .map(|f| f.data_type().clone())
+ .collect::<Vec<_>>();
+ // The constructed struct row is never NULL: `invoke_with_args` builds
+ // the `StructArray` without a null buffer, so the output field is
+ // non-nullable and `struct(...) IS NOT NULL` folds to `true`.
+ Ok(Field::new(self.name(), self.return_type(&arg_types)?, false).into())
+ }
+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let DataType::Struct(fields) = args.return_type() else {
return internal_err!("incorrect struct return type");
diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt
index 183bfbd04e..7704b7f214 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -1752,3 +1752,53 @@ limit 2;
statement ok
drop table list_cast_limit;
+
+# named_struct() and struct() never produce a NULL row, so their return field
+# is non-nullable and `IS NOT NULL` on the constructor folds to `true`. A guard
+# on a struct built by a view then no longer keeps the whole struct alive, and
+# the scan is pruned to the fields that are actually read.
+statement ok
+create table struct_ctor_null (a int, b int, c int) as values (1, 2, 3), (NULL, 5, 6);
+
+statement ok
+create view struct_ctor_view as select named_struct('a', a, 'b', b, 'c', c) as s from struct_ctor_null;
+
+query BB
+select named_struct('a', a) is not null, struct(a) is null from struct_ctor_null;
+----
+true false
+true false
+
+query TT
+explain select s['b'] from struct_ctor_view where s is not null;
+----
+logical_plan
+01)Projection: __datafusion_extracted_1 AS struct_ctor_view.s[b]
+02)--SubqueryAlias: struct_ctor_view
+03)----Projection: struct_ctor_null.b AS __datafusion_extracted_1
+04)------TableScan: struct_ctor_null projection=[b]
+physical_plan
+01)ProjectionExec: expr=[b@0 as struct_ctor_view.s[b]]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+select s['b'] from struct_ctor_view where s is not null;
+----
+2
+5
+
+query TT
+explain select case when named_struct('a', a) is not null then b end from struct_ctor_null;
+----
+logical_plan
+01)Projection: struct_ctor_null.b AS CASE WHEN named_struct(Utf8("a"),struct_ctor_null.a) IS NOT NULL THEN struct_ctor_null.b END
+02)--TableScan: struct_ctor_null projection=[b]
+physical_plan
+01)ProjectionExec: expr=[b@0 as CASE WHEN named_struct(Utf8("a"),struct_ctor_null.a) IS NOT NULL THEN struct_ctor_null.b END]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop view struct_ctor_view;
+
+statement ok
+drop table struct_ctor_null;