Apache DataFusion SQL Query Engine
fix: Handle nulls correctly when extracting nested arrays from nullable struct/map (#25122)
Adam Gutglick140c7c5a4cbd
diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rsindex 475c6518d2..a7526dc5c0 100644--- a/datafusion/functions/src/core/getfield.rs+++ b/datafusion/functions/src/core/getfield.rs@@ -18,10 +18,12 @@use std::sync::{Arc, OnceLock};use arrow::array::{- Array, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array,+ Array, ArrayRef, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array,make_comparator,};-use arrow::compute::SortOptions;+use arrow::array::{BooleanArray, layout};+use arrow::buffer::NullBuffer;+use arrow::compute::{SortOptions, nullif};use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::cast::{as_map_array, as_struct_array};@@ -125,6 +127,10 @@ fn process_map_array(let matches = keys.values();for entry in 0..map_array.len() {+ if map_array.is_null(entry) {+ mutable.try_extend_nulls(1)?;+ continue;+ }let start = offsets[entry] as usize;let end = offsets[entry + 1] as usize;@@ -160,6 +166,10 @@ fn process_map_with_nested_key(MutableArrayData::with_capacities(vec![&original_data], true, capacity);for entry in 0..map_array.len() {+ if map_array.is_null(entry) {+ mutable.try_extend_nulls(1)?;+ continue;+ }let start = map_array.value_offsets()[entry] as usize;let end = map_array.value_offsets()[entry + 1] as usize;@@ -182,6 +192,46 @@ fn process_map_with_nested_key(Ok(ColumnarValue::Array(data))}+/// Apply a struct's nulls to one of its fields.+fn apply_parent_nulls(+ col: &ArrayRef,+ parent_nulls: Option<&NullBuffer>,+) -> Result<ArrayRef> {+ let Some(parent_nulls) = parent_nulls else {+ // If there are no parent nulls to apply, we can just return+ return Ok(Arc::clone(col));+ };++ // NullArray is already entirely null and cannot have a validity bitmap.+ // If we have 0 parent nulls, we can also avoid extra work.+ if col.data_type().is_null() || parent_nulls.null_count() == 0 {+ return Ok(Arc::clone(col));+ }++ if layout(col.data_type()).can_contain_null_mask {+ // `nullif` marks a row null where the mask is true and keeps the+ // field's own nulls. Only the validity bitmap is rebuilt; the value+ // buffers and child arrays are shared with `col`.+ let null_parents = BooleanArray::new(!parent_nulls.inner(), None);+ return Ok(nullif(col.as_ref(), &null_parents)?);+ }++ // Unions and run-end encoded arrays have no validity bitmap of their own+ // and represent nulls in their children. Rebuild the array so null parents+ // become null values in those children.+ let data = col.to_data();+ let mut mutable = MutableArrayData::new(vec![&data], true, data.len());+ let mut end = 0;+ for (start, valid_end) in parent_nulls.valid_slices() {+ mutable.try_extend_nulls(start - end)?;+ mutable.try_extend(0, start, valid_end)?;+ end = valid_end;+ }+ mutable.try_extend_nulls(data.len() - end)?;++ Ok(make_array(mutable.freeze()))+}+/// Extract a single field from a struct or map arrayfn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<ColumnarValue> {let arrays = ColumnarValue::values_to_arrays(&[base])?;@@ -204,9 +254,8 @@ fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<Column"Field {field_name} not found in dictionary struct")})?;- Ok(ColumnarValue::Array(- dict.with_values(Arc::clone(field_col)),- ))+ let field_col = apply_parent_nulls(field_col, values_struct.nulls())?;+ Ok(ColumnarValue::Array(dict.with_values(field_col)))}(DataType::Map(_, _), key, _) => {// The lookup key is a single scalar. `eq` does not support nested@@ -222,7 +271,10 @@ fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<Columnlet as_struct_array = as_struct_array(&array)?;match as_struct_array.column_by_name(&k) {None => exec_err!("Field {k} not found in struct"),- Some(col) => Ok(ColumnarValue::Array(Arc::clone(col))),+ Some(col) => Ok(ColumnarValue::Array(apply_parent_nulls(+ col,+ as_struct_array.nulls(),+ )?)),}}(DataType::Struct(_), name, _) => exec_err!(@@ -657,10 +709,10 @@ impl ScalarUDFImpl for GetFieldFunc {mod tests {use super::*;use arrow::array::{- ArrayRef, Int32Array, Int32Builder, ListArray, ListBuilder, MapBuilder,- StructArray,+ ArrayRef, Int32Array, Int32Builder, ListArray, ListBuilder, MapBuilder, RunArray,+ StructArray, UnionArray,};- use arrow::datatypes::{Fields, Int32Type};+ use arrow::datatypes::{Fields, Int32Type, UnionFields};#[test]fn test_get_field_utf8view_key() -> Result<()> {@@ -733,6 +785,213 @@ mod tests {Ok(())}+ #[test]+ fn test_get_field_nested_struct_outer_nulls() -> Result<()> {+ let inner_array = StructArray::new(+ vec![Field::new("value", DataType::Int32, false)].into(),+ vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],+ None,+ );++ // Only the outer struct marks row 1 as null; its children are all valid.+ let outer_array = StructArray::new(+ vec![Field::new("inner", inner_array.data_type().clone(), false)].into(),+ vec![Arc::new(inner_array)],+ Some(NullBuffer::from(vec![true, false, true])),+ );++ for (input, expected) in [+ (outer_array.clone(), vec![Some(1), None, Some(3)]),+ (outer_array.slice(1, 2), vec![None, Some(3)]),+ ] {+ let inner = extract_single_field(+ ColumnarValue::Array(Arc::new(input)),+ ScalarValue::Utf8(Some("inner".to_string())),+ )?;+ let result = extract_single_field(+ inner,+ ScalarValue::Utf8(Some("value".to_string())),+ )?+ .into_array(expected.len())?;++ let expected = Int32Array::from(expected);+ assert_eq!(result.as_ref(), &expected as &dyn Array);+ }++ Ok(())+ }++ #[test]+ fn test_get_field_map_parent_nulls() -> Result<()> {+ use arrow::array::{FixedSizeListArray, MapArray};+ use arrow_buffer::OffsetBuffer;++ let keys = Arc::new(Int32Array::from(vec![7; 3])) as ArrayRef;+ let nested_keys = Arc::new(FixedSizeListArray::new(+ Arc::new(Field::new("item", DataType::Int32, false)),+ 1,+ Arc::clone(&keys),+ None,+ )) as ArrayRef;++ // Exercise both map lookup paths. The null map has a valid matching entry.+ for keys in [keys, nested_keys] {+ let key = ScalarValue::try_from_array(keys.as_ref(), 0)?;+ let entries = StructArray::new(+ vec![+ Field::new("key", keys.data_type().clone(), false),+ Field::new("value", DataType::Int32, false),+ ]+ .into(),+ vec![keys, Arc::new(Int32Array::from(vec![1, 2, 3]))],+ None,+ );+ let map = MapArray::new(+ Arc::new(Field::new("entries", entries.data_type().clone(), false)),+ OffsetBuffer::new(vec![0, 1, 2, 3].into()),+ entries,+ Some(NullBuffer::from(vec![true, false, true])),+ false,+ );+ let result = extract_single_field(ColumnarValue::Array(Arc::new(map)), key)?+ .into_array(3)?;+ let expected = Int32Array::from(vec![Some(1), None, Some(3)]);+ assert_eq!(result.as_ref(), &expected as &dyn Array);+ }+ Ok(())+ }++ #[test]+ fn test_get_field_null_typed_child() -> Result<()> {+ use arrow::array::{DictionaryArray, NullArray, UInt32Array};+ use arrow::datatypes::UInt32Type;++ let values = Arc::new(StructArray::new(+ vec![Field::new("value", DataType::Null, true)].into(),+ vec![Arc::new(NullArray::new(2))],+ Some(NullBuffer::from(vec![true, false])),+ )) as ArrayRef;+ let dictionary = Arc::new(DictionaryArray::<UInt32Type>::try_new(+ UInt32Array::from(vec![0, 1]),+ Arc::clone(&values),+ )?) as ArrayRef;++ for input in [values, dictionary] {+ let result = extract_single_field(+ ColumnarValue::Array(input),+ ScalarValue::Utf8(Some("value".to_string())),+ )?+ .into_array(2)?;+ assert_eq!(result.logical_null_count(), 2);+ }+ Ok(())+ }++ #[test]+ fn test_get_field_union_parent_nulls() -> Result<()> {+ use arrow::array::{DictionaryArray, StringArray, UInt32Array};+ use arrow::datatypes::UInt32Type;++ let fields = UnionFields::try_new(+ [3, 7],+ [+ Field::new("int", DataType::Int32, true),+ Field::new("string", DataType::Utf8, true),+ ],+ )?;+ let ints = Int32Array::from(vec![Some(9), Some(1), Some(2), None, Some(4)]);+ let strings = StringArray::from(vec!["x"; 5]);++ // Dense rows 1 and 2 share a value, but only row 2 has a null parent.+ for offsets in [None, Some(vec![0, 1, 1, 3, 0].into())] {+ let child = UnionArray::try_new(+ fields.clone(),+ vec![7, 3, 3, 3, 7].into(),+ offsets,+ vec![Arc::new(ints.clone()), Arc::new(strings.clone())],+ )?;+ let union_type = child.data_type().clone();+ let parent = StructArray::new(+ vec![Field::new("u", union_type.clone(), true)].into(),+ vec![Arc::new(child)],+ Some(NullBuffer::from(vec![true, true, false, true, true])),+ );+ let values = Arc::new(parent.slice(1, 4)) as ArrayRef;+ let dictionary = Arc::new(DictionaryArray::<UInt32Type>::try_new(+ UInt32Array::from(vec![0, 1, 2, 3]),+ Arc::clone(&values),+ )?) as ArrayRef;++ for input in [values, dictionary] {+ let result = extract_single_field(+ ColumnarValue::Array(input),+ ScalarValue::Utf8(Some("u".to_string())),+ )?+ .into_array(4)?;+ assert_eq!(+ result.logical_nulls(),+ Some(NullBuffer::from(vec![true, false, false, true]))+ );+ let result = match result.data_type() {+ DataType::Dictionary(_, _) => result.as_any_dictionary().values(),+ _ => &result,+ };+ assert_eq!(result.data_type(), &union_type);+ result.to_data().validate_full()?;+ let result = result.as_union();+ assert_eq!(result.value(0).as_ref(), &Int32Array::from(vec![1]));+ assert_eq!(result.value(3).as_ref(), &StringArray::from(vec!["x"]));+ }+ }+ Ok(())+ }++ #[test]+ fn test_get_field_run_encoded_parent_nulls() -> Result<()> {+ // Logical values: [10, 10, 20, NULL, NULL]+ let run_ends = Int32Array::from(vec![2, 3, 5]);+ let values = Int32Array::from(vec![Some(10), Some(20), None]);+ let child = RunArray::<Int32Type>::try_new(&run_ends, &values)?;+ let run_type = child.data_type().clone();+ let parent = StructArray::new(+ vec![Field::new("r", run_type.clone(), true)].into(),+ vec![Arc::new(child)],+ Some(NullBuffer::from(vec![true, false, true, true, true])),+ );++ for (input, expected) in [+ (+ Arc::new(parent.clone()) as ArrayRef,+ vec![Some(10), None, Some(20), None, None],+ ),+ (+ Arc::new(parent.slice(1, 4)) as ArrayRef,+ vec![None, Some(20), None, None],+ ),+ ] {+ let result = extract_single_field(+ ColumnarValue::Array(input),+ ScalarValue::Utf8(Some("r".to_string())),+ )?+ .into_array(expected.len())?;+ assert_eq!(result.data_type(), &run_type);+ result.to_data().validate_full()?;++ let run_array = result.as_run::<Int32Type>();+ let run_values = run_array.values().as_primitive::<Int32Type>();+ let logical: Vec<Option<i32>> = (0..run_array.len())+ .map(|i| {+ let physical = run_array.get_physical_index(i);+ run_values+ .is_valid(physical)+ .then(|| run_values.value(physical))+ })+ .collect();+ assert_eq!(logical, expected);+ }+ Ok(())+ }+#[test]fn test_get_field_dict_encoded_struct() -> Result<()> {use arrow::array::{DictionaryArray, StringArray, UInt32Array};diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 87d39e34dc..76dcb322a9 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -812,23 +812,25 @@ fn register_dictionary_struct_table(ctx: &SessionContext) {ctx.register_batch("dict_struct_table", batch).unwrap();- // Second table: dictionary-encoded struct with nullable entries- let names_nullable = Arc::new(StringArray::from(vec!["X", "Y"])) as ArrayRef;- let ids_nullable = Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef;+ // Second table: null keys, null structs with valid children, and null children.+ let names_nullable =+ Arc::new(StringArray::from(vec!["X", "Y", "hidden"])) as ArrayRef;+ let ids_nullable =+ Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef;let struct_fields_nullable: Fields = vec![Field::new("name", DataType::Utf8, false),- Field::new("id", DataType::Int32, false),+ Field::new("id", DataType::Int32, true),].into();let values_struct_nullable = Arc::new(StructArray::try_new(struct_fields_nullable.clone(),vec![names_nullable, ids_nullable],- None,+ Some(vec![true, true, false].into()),).unwrap(),) as ArrayRef;- let keys_nullable = UInt32Array::from(vec![Some(0), None, Some(1), None]);+ let keys_nullable = UInt32Array::from(vec![Some(0), None, Some(1), Some(2), Some(2)]);let dict_nullable =DictionaryArray::<UInt32Type>::try_new(keys_nullable, values_struct_nullable).unwrap();diff --git a/datafusion/sqllogictest/test_files/dictionary_struct.slt b/datafusion/sqllogictest/test_files/dictionary_struct.sltindex d97cb5420e..e2ddc8be1c 100644--- a/datafusion/sqllogictest/test_files/dictionary_struct.slt+++ b/datafusion/sqllogictest/test_files/dictionary_struct.slt@@ -33,12 +33,14 @@# {name: Bob, id: 2}## dict_struct_nullable:-# ds Dictionary(UInt32, Struct(name: Utf8, id: Int32)) — 4 rows, keys [0, NULL, 1, NULL]+# ds Dictionary(UInt32, Struct(name: Utf8, id: Int32)) — 5 rows, keys [0, NULL, 1, 2, 2]+# Value 2 is a null struct with valid child values ("hidden", 30).## Rows (logical values):# {name: X, id: 10}# NULL-# {name: Y, id: 20}+# {name: Y, id: NULL}+# NULL# NULL# Verify schema of dict_struct_table@@ -67,11 +69,11 @@ SELECT dict_struct['id'] FROM dict_struct_table;12-# Verify the extracted field preserves dictionary encoding-query T-SELECT arrow_typeof(dict_struct['name']) FROM dict_struct_table LIMIT 1;+# Verify the extracted field preserves dictionary encoding and nullability+query TBB+SELECT arrow_typeof(dict_struct['name']), arrow_field(dict_struct['name'])['nullable'], arrow_field(plain_struct['name'])['nullable'] FROM dict_struct_table LIMIT 1;-----Dictionary(UInt32, Utf8)+Dictionary(UInt32, Utf8) false falsequery TSELECT arrow_typeof(dict_struct['id']) FROM dict_struct_table LIMIT 1;@@ -107,21 +109,23 @@ CarolAliceBob-# Field extraction from dict-encoded struct with NULLs-query T-SELECT ds['name'] FROM dict_struct_nullable;+# A nullable parent makes its non-nullable child nullable, preserving the dictionary type.+query TTB+SELECT ds['name'], arrow_typeof(ds['name']), arrow_field(ds['name'])['nullable'] FROM dict_struct_nullable;-----X-NULL-Y-NULL+X Dictionary(UInt32, Utf8) true+NULL Dictionary(UInt32, Utf8) true+Y Dictionary(UInt32, Utf8) true+NULL Dictionary(UInt32, Utf8) true+NULL Dictionary(UInt32, Utf8) truequery ?SELECT ds['id'] FROM dict_struct_nullable;----10NULL-20+NULL+NULLNULL# Filtering on extracted dict-encoded struct fielddiff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.sltindex debaf7f8b3..b69f04defc 100644--- a/datafusion/sqllogictest/test_files/map.slt+++ b/datafusion/sqllogictest/test_files/map.slt@@ -87,10 +87,11 @@ GET 27PUT 25DELETE 24-query T-SELECT strings['not_found'] FROM data LIMIT 1;+# A missing key makes the result nullable even when the map and its values are not.+query TBB+SELECT strings['not_found'], arrow_field(strings)['nullable'], arrow_field(strings['not_found'])['nullable'] FROM data LIMIT 1;-----NULL+NULL false true# Select non existent key, expect NULL for each rowquery Idiff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex 7704b7f214..87bbd11c98 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -766,22 +766,23 @@ select get_field(CAST(NULL AS STRUCT(a STRUCT(b INT))), 'a', 'b');----NULL-# Null handling: null in middle of chain+# Null parents at either level make a non-nullable child nullable.statement ok-create table null_mid_test (s STRUCT(a STRUCT(b INT)));--statement ok-insert into null_mid_test values ({'a': NULL});+create table null_mid_test as+select arrow_cast(NULL, 'Struct("a": non-null Struct("b": non-null Int32))') as null_base,+ arrow_cast({'a': NULL}, 'Struct("a": Struct("b": non-null Int32))') as s;query Iselect s['a']['b'] from null_mid_test;----NULL-query I-select get_field(s, 'a', 'b') from null_mid_test;+query IBIB+select get_field(null_base, 'a', 'b'), arrow_field(null_base['a']['b'])['nullable'],+ get_field(s, 'a', 'b'), arrow_field(s['a']['b'])['nullable']+from null_mid_test;-----NULL+NULL true NULL truestatement okdrop table null_mid_test;