Apache DataFusion SQL Query Engine
feat: Add support for `explode_outer` function for arrays (#22100)
Krishna Sudarshan Jf2b483598b23
diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rsindex 2f6d9848b6..2eebfe4963 100644--- a/datafusion/common/src/lib.rs+++ b/datafusion/common/src/lib.rs@@ -99,7 +99,7 @@ pub use schema_reference::SchemaReference;pub use spans::{Location, Span, Spans};pub use stats::{ColumnStatistics, Statistics};pub use table_reference::{ResolvedTableReference, TableReference};-pub use unnest::{RecursionUnnestOption, UnnestOptions};+pub use unnest::{NullHandling, RecursionUnnestOption, UnnestOptions};pub use utils::project_schema;// These are hidden from docs purely to avoid polluting the public view of what this crate exports.diff --git a/datafusion/common/src/unnest.rs b/datafusion/common/src/unnest.rsindex db48edd061..58aed390ac 100644--- a/datafusion/common/src/unnest.rs+++ b/datafusion/common/src/unnest.rs@@ -19,23 +19,38 @@use crate::Column;+/// How [`UnnestOptions`] handles `NULL` and empty list values in the input column.+///+/// The variants enumerate the three observable behaviors so that callers do+/// not have to compose multiple boolean flags to express what they want.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Hash)]+pub enum NullHandling {+ /// Drop rows where the input list is `NULL` or empty. Matches the+ /// default behavior of systems such as DuckDB and ClickHouse.+ Drop,+ /// Preserve `NULL` input rows as a single output row containing `NULL`.+ /// Empty lists still produce zero output rows. This is the default and+ /// matches DataFusion's historical `preserve_nulls = true` behavior.+ #[default]+ Preserve,+ /// Like [`Self::Preserve`], and additionally treat an empty list+ /// identically to a `NULL` list, producing a single output row+ /// containing `NULL`.+ PreserveAndExpandEmpty,+}+/// Options for unnesting a column that contains a list type,/// replicating values in the other, non nested rows.////// Conceptually this operation is like joining each row with all the/// values in the list column.///-/// If `preserve_nulls` is false, nulls and empty lists-/// from the input column are not carried through to the output. This-/// is the default behavior for other systems such as ClickHouse and-/// DuckDB-///-/// If `preserve_nulls` is true (the default), nulls from the input-/// column are carried through to the output.+/// The behavior with `NULL` and empty input lists is controlled by+/// [`NullHandling`]. See its variants for full details.////// # Examples///-/// ## `Unnest(c1)`, preserve_nulls: false+/// ## `Unnest(c1)`, null_handling: NullHandling::Drop/// ```text/// ┌─────────┐ ┌─────┐ ┌─────────┐ ┌─────┐/// │ {1, 2} │ │ A │ Unnest │ 1 │ │ A │@@ -49,7 +64,7 @@ use crate::Column;/// c1 c2/// ```///-/// ## `Unnest(c1)`, preserve_nulls: true+/// ## `Unnest(c1)`, null_handling: NullHandling::Preserve/// ```text/// ┌─────────┐ ┌─────┐ ┌─────────┐ ┌─────┐/// │ {1, 2} │ │ A │ Unnest │ 1 │ │ A │@@ -63,13 +78,30 @@ use crate::Column;/// c1 c2 c1 c2/// ```///+/// ## `Unnest(c1)`, null_handling: NullHandling::PreserveAndExpandEmpty+/// ```text+/// ┌─────────┐ ┌─────┐ ┌─────────┐ ┌─────┐+/// │ {1, 2} │ │ A │ Unnest │ 1 │ │ A │+/// ├─────────┤ ├─────┤ ├─────────┤ ├─────┤+/// │ null │ │ B │ │ 2 │ │ A │+/// ├─────────┤ ├─────┤ ────────────▶ ├─────────┤ ├─────┤+/// │ {} │ │ D │ │ null │ │ B │+/// ├─────────┤ ├─────┤ ├─────────┤ ├─────┤+/// │ {3} │ │ E │ │ null │ │ D │+/// └─────────┘ └─────┘ ├─────────┤ ├─────┤+/// c1 c2 │ 3 │ │ E │+/// └─────────┘ └─────┘+/// c1 c2+/// ```+////// `recursions` instruct how a column should be unnested (e.g unnesting a column multiple/// time, with depth = 1 and depth = 2). Any unnested column not being mentioned inside this/// options is inferred to be unnested with depth = 1#[derive(Debug, Clone, PartialEq, PartialOrd, Hash, Eq)]pub struct UnnestOptions {- /// Should nulls in the input be preserved? Defaults to true- pub preserve_nulls: bool,+ /// How to handle `NULL` and empty list values in the input column.+ /// Defaults to [`NullHandling::Preserve`].+ pub null_handling: NullHandling,/// If specific columns need to be unnested multiple times (e.g at different depth),/// declare them here. Any unnested columns not being mentioned inside this option/// will be unnested with depth = 1@@ -88,8 +120,7 @@ pub struct RecursionUnnestOption {impl Default for UnnestOptions {fn default() -> Self {Self {- // default to true to maintain backwards compatible behavior- preserve_nulls: true,+ null_handling: NullHandling::Preserve,recursions: vec![],}}@@ -101,13 +132,41 @@ impl UnnestOptions {Default::default()}- /// Set the behavior with nulls in the input as described on- /// [`Self`]- pub fn with_preserve_nulls(mut self, preserve_nulls: bool) -> Self {- self.preserve_nulls = preserve_nulls;+ /// Set the [`NullHandling`] mode used when unnesting `NULL` or empty+ /// input lists.+ pub fn with_null_handling(mut self, null_handling: NullHandling) -> Self {+ self.null_handling = null_handling;self}+ /// Backward-compatible setter that maps the previous boolean+ /// `preserve_nulls` flag onto [`NullHandling`].+ ///+ /// `true` maps to [`NullHandling::Preserve`]; `false` maps to+ /// [`NullHandling::Drop`]. To opt into the new empty-list-preserving+ /// mode, call [`Self::with_null_handling`] directly with+ /// [`NullHandling::PreserveAndExpandEmpty`].+ pub fn with_preserve_nulls(self, preserve_nulls: bool) -> Self {+ let null_handling = if preserve_nulls {+ NullHandling::Preserve+ } else {+ NullHandling::Drop+ };+ self.with_null_handling(null_handling)+ }++ /// Returns true if `NULL` input rows produce a single output row+ /// containing `NULL`.+ pub fn preserve_nulls(&self) -> bool {+ !matches!(self.null_handling, NullHandling::Drop)+ }++ /// Returns true if empty input lists should produce a single+ /// output row containing `NULL`.+ pub fn expand_empty_as_null(&self) -> bool {+ matches!(self.null_handling, NullHandling::PreserveAndExpandEmpty)+ }+/// Set the recursions for the unnest operationpub fn with_recursions(mut self, recursion: RecursionUnnestOption) -> Self {self.recursions.push(recursion);diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex b9fecb5fdd..73a9177ab7 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -4370,6 +4370,28 @@ async fn unnest_column_nulls() -> Result<()> {);let options = UnnestOptions::new().with_preserve_nulls(false);+ let results = df+ .clone()+ .unnest_columns_with_options(&["list"], options)?+ .collect()+ .await?;+ assert_snapshot!(+ batches_to_string(&results),+ @r"+ +------+----++ | list | id |+ +------+----++ | 1 | A |+ | 2 | A |+ | 3 | D |+ +------+----++ "+ );++ // Outer-unnest semantics: NULL and empty lists both produce a single+ // output row containing NULL.+ let options = UnnestOptions::new()+ .with_null_handling(datafusion_common::NullHandling::PreserveAndExpandEmpty);let results = df.unnest_columns_with_options(&["list"], options)?.collect()@@ -4382,6 +4404,8 @@ async fn unnest_column_nulls() -> Result<()> {+------+----+| 1 | A || 2 | A |+ | | B |+ | | C || 3 | D |+------+----+"@@ -4390,6 +4414,133 @@ async fn unnest_column_nulls() -> Result<()> {Ok(())}+/// Outer-unnest on a list-of-struct column. Verifies that+/// (a) struct elements unnest into flattened sub-columns and+/// (b) NULL and empty lists both still produce a single output row whose+/// struct sub-columns are all NULL.+#[tokio::test]+async fn unnest_outer_list_of_struct() -> Result<()> {+ use arrow::array::{Int32Array, StructArray};++ // Per-row sub-list lengths: 2, 1, 0 (empty), 0 (null)+ let names = StringArray::from(vec!["alice", "bob", "carol"]);+ let ages = Int32Array::from(vec![30, 40, 50]);+ let struct_values = StructArray::from(vec![+ (+ Arc::new(Field::new("name", DataType::Utf8, true)),+ Arc::new(names) as ArrayRef,+ ),+ (+ Arc::new(Field::new("age", DataType::Int32, true)),+ Arc::new(ages) as ArrayRef,+ ),+ ]);+ let struct_field =+ Arc::new(Field::new("item", struct_values.data_type().clone(), true));+ let offsets = arrow::buffer::OffsetBuffer::<i32>::from_lengths([2, 1, 0, 0]);+ let validity = arrow::buffer::NullBuffer::from(vec![true, true, true, false]);+ let people = ListArray::new(+ struct_field,+ offsets,+ Arc::new(struct_values),+ Some(validity),+ );+ let group = Int32Array::from(vec![1, 2, 3, 4]);++ let batch = RecordBatch::try_from_iter(vec![+ ("people", Arc::new(people) as ArrayRef),+ ("group", Arc::new(group) as ArrayRef),+ ])?;++ let ctx = SessionContext::new();+ ctx.register_batch("teams", batch)?;+ let df = ctx.table("teams").await?;++ let options = UnnestOptions::new()+ .with_null_handling(datafusion_common::NullHandling::PreserveAndExpandEmpty);+ let results = df+ // Unnest the list, then expand the resulting struct rows into columns.+ .unnest_columns_with_options(&["people"], options.clone())?+ .unnest_columns_with_options(&["people"], options)?+ .collect()+ .await?;+ assert_snapshot!(+ batches_to_string(&results),+ @r"+ +-------------+------------+-------++ | people.name | people.age | group |+ +-------------+------------+-------++ | alice | 30 | 1 |+ | bob | 40 | 1 |+ | carol | 50 | 2 |+ | | | 3 |+ | | | 4 |+ +-------------+------------+-------++ "+ );++ Ok(())+}++/// Outer-unnest applied to a `FixedSizeList` column. For fixed-size lists,+/// every non-null row has the fixed length, so "empty" never occurs —+/// `PreserveAndExpandEmpty` should behave identically to `Preserve` here.+/// The test pins that equivalence so we notice if it ever diverges.+#[tokio::test]+async fn unnest_outer_fixed_size_list() -> Result<()> {+ let batch = get_fixed_list_batch()?;+ let ctx = SessionContext::new();+ ctx.register_batch("shapes", batch)?;+ let df = ctx.table("shapes").await?;++ let preserve_results = df+ .clone()+ .unnest_columns_with_options(+ &["tags"],+ UnnestOptions::new().with_preserve_nulls(true),+ )?+ .collect()+ .await?;+ let outer_results = df+ .unnest_columns_with_options(+ &["tags"],+ UnnestOptions::new().with_null_handling(+ datafusion_common::NullHandling::PreserveAndExpandEmpty,+ ),+ )?+ .collect()+ .await?;+ assert_eq!(+ batches_to_sort_string(&preserve_results),+ batches_to_sort_string(&outer_results),+ "FixedSizeList has no empty case, so PreserveAndExpandEmpty must \+ match Preserve exactly"+ );++ // And the snapshot itself, to make the expected shape explicit.+ assert_snapshot!(+ batches_to_sort_string(&outer_results),+ @r"+ +----------+-------++ | shape_id | tags |+ +----------+-------++ | 1 | |+ | 2 | tag21 |+ | 2 | tag22 |+ | 3 | tag31 |+ | 3 | tag32 |+ | 4 | |+ | 5 | tag51 |+ | 5 | tag52 |+ | 6 | tag61 |+ | 6 | tag62 |+ +----------+-------++ "+ );++ Ok(())+}+#[tokio::test]async fn unnest_fixed_list() -> Result<()> {let batch = get_fixed_list_batch()?;diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rsindex b6ffd74ea2..f9c0662e68 100644--- a/datafusion/expr/src/expr.rs+++ b/datafusion/expr/src/expr.rs@@ -671,22 +671,43 @@ pub fn intersect_metadata_for_union<'a>(}/// UNNEST expression.+///+/// When `outer` is `true`, the unnest should preserve `NULL` and empty input+/// lists by emitting a single `NULL` output row for each. When `false` (the+/// historical default), the behavior is identical to the plain `UNNEST(col)`+/// SQL form: `NULL` and empty input lists are dropped from the output.#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]pub struct Unnest {pub expr: Box<Expr>,+ /// Outer-unnest behavior: also expand empty input lists into a single+ /// `NULL` output row (in addition to preserving `NULL` input rows).+ pub outer: bool,}impl Unnest {- /// Create a new Unnest expression.+ /// Create a new Unnest expression with default (non-outer) semantics.pub fn new(expr: Expr) -> Self {Self {expr: Box::new(expr),+ outer: false,}}- /// Create a new Unnest expression.+ /// Create a new Unnest expression with default (non-outer) semantics.pub fn new_boxed(boxed: Box<Expr>) -> Self {- Self { expr: boxed }+ Self {+ expr: boxed,+ outer: false,+ }+ }++ /// Create a new Unnest expression with outer-unnest semantics: `NULL`+ /// and empty input lists each produce a single `NULL` output row.+ pub fn new_outer(expr: Expr) -> Self {+ Self {+ expr: Box::new(expr),+ outer: true,+ }}}@@ -2431,11 +2452,19 @@ impl NormalizeEq for Expr {| (Expr::IsNotTrue(self_expr), Expr::IsNotTrue(other_expr))| (Expr::IsNotFalse(self_expr), Expr::IsNotFalse(other_expr))| (Expr::IsNotUnknown(self_expr), Expr::IsNotUnknown(other_expr))- | (Expr::Negative(self_expr), Expr::Negative(other_expr))- | (- Expr::Unnest(Unnest { expr: self_expr }),- Expr::Unnest(Unnest { expr: other_expr }),- ) => self_expr.normalize_eq(other_expr),+ | (Expr::Negative(self_expr), Expr::Negative(other_expr)) => {+ self_expr.normalize_eq(other_expr)+ }+ (+ Expr::Unnest(Unnest {+ expr: self_expr,+ outer: self_outer,+ }),+ Expr::Unnest(Unnest {+ expr: other_expr,+ outer: other_outer,+ }),+ ) => self_outer == other_outer && self_expr.normalize_eq(other_expr),(Expr::Between(Between {expr: self_expr,@@ -2883,7 +2912,9 @@ impl HashNode for Expr {field.hash(state);column.hash(state);}- Expr::Unnest(Unnest { expr: _expr }) => {}+ Expr::Unnest(Unnest { expr: _expr, outer }) => {+ outer.hash(state);+ }Expr::HigherOrderFunction(HigherOrderFunction { func, args: _args }) => {func.hash(state);}@@ -3123,8 +3154,9 @@ impl Display for SchemaDisplay<'_> {}Expr::Negative(expr) => write!(f, "(- {})", SchemaDisplay(expr)),Expr::Not(expr) => write!(f, "NOT {}", SchemaDisplay(expr)),- Expr::Unnest(Unnest { expr }) => {- write!(f, "UNNEST({})", SchemaDisplay(expr))+ Expr::Unnest(Unnest { expr, outer }) => {+ let name = if *outer { "UNNEST_OUTER" } else { "UNNEST" };+ write!(f, "{name}({})", SchemaDisplay(expr))}Expr::ScalarFunction(ScalarFunction { func, args }) => {match func.schema_name(args) {@@ -3398,8 +3430,9 @@ impl Display for SqlDisplay<'_> {}Expr::Negative(expr) => write!(f, "(- {})", SqlDisplay(expr)),Expr::Not(expr) => write!(f, "NOT {}", SqlDisplay(expr)),- Expr::Unnest(Unnest { expr }) => {- write!(f, "UNNEST({})", SqlDisplay(expr))+ Expr::Unnest(Unnest { expr, outer }) => {+ let name = if *outer { "UNNEST_OUTER" } else { "UNNEST" };+ write!(f, "{name}({})", SqlDisplay(expr))}Expr::SimilarTo(Like {negated,@@ -3752,7 +3785,7 @@ impl Display for Expr {}},Expr::Placeholder(Placeholder { id, .. }) => write!(f, "{id}"),- Expr::Unnest(Unnest { expr }) => {+ Expr::Unnest(Unnest { expr, .. }) => {write!(f, "{UNNEST_COLUMN_PREFIX}({expr})")}Expr::HigherOrderFunction(fun) => {diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rsindex 9d711113e4..b1a5a12d15 100644--- a/datafusion/expr/src/expr_fn.rs+++ b/datafusion/expr/src/expr_fn.rs@@ -386,10 +386,11 @@ pub fn when(when: Expr, then: Expr) -> CaseBuilder {CaseBuilder::new(None, vec![when], vec![then], None)}-/// Create a Unnest expression+/// Create a Unnest expression with default (non-outer) semantics.pub fn unnest(expr: Expr) -> Expr {Expr::Unnest(Unnest {expr: Box::new(expr),+ outer: false,})}diff --git a/datafusion/expr/src/expr_rewriter/mod.rs b/datafusion/expr/src/expr_rewriter/mod.rsindex a9a0c15653..7a6ac3fc8b 100644--- a/datafusion/expr/src/expr_rewriter/mod.rs+++ b/datafusion/expr/src/expr_rewriter/mod.rs@@ -87,13 +87,16 @@ pub fn normalize_col_with_schemas_and_ambiguity_check(using_columns: &[HashSet<Column>],) -> Result<Expr> {// Normalize column inside Unnest- if let Expr::Unnest(Unnest { expr }) = expr {+ if let Expr::Unnest(Unnest { expr, outer }) = expr {let e = normalize_col_with_schemas_and_ambiguity_check(expr.as_ref().clone(),schemas,using_columns,)?;- return Ok(Expr::Unnest(Unnest { expr: Box::new(e) }));+ return Ok(Expr::Unnest(Unnest {+ expr: Box::new(e),+ outer,+ }));}expr.transform(|expr| {diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rsindex 039bbad65a..ec367de846 100644--- a/datafusion/expr/src/expr_schema.rs+++ b/datafusion/expr/src/expr_schema.rs@@ -157,7 +157,7 @@ impl ExprSchemable for Expr {Expr::Cast(Cast { field, .. }) | Expr::TryCast(TryCast { field, .. }) => {Ok(field.data_type().clone())}- Expr::Unnest(Unnest { expr }) => {+ Expr::Unnest(Unnest { expr, .. }) => {let arg_data_type = expr.get_type(schema)?;// Unnest's output type is the inner type of the listmatch arg_data_type {diff --git a/datafusion/expr/src/tree_node.rs b/datafusion/expr/src/tree_node.rsindex 010441b5a2..941fd22ea1 100644--- a/datafusion/expr/src/tree_node.rs+++ b/datafusion/expr/src/tree_node.rs@@ -49,7 +49,7 @@ impl TreeNode for Expr {) -> Result<TreeNodeRecursion> {match self {Expr::Alias(Alias { expr, .. })- | Expr::Unnest(Unnest { expr })+ | Expr::Unnest(Unnest { expr, .. })| Expr::Not(expr)| Expr::IsNotNull(expr)| Expr::IsTrue(expr)@@ -150,9 +150,9 @@ impl TreeNode for Expr {quantifier,})}),- Expr::Unnest(Unnest { expr, .. }) => expr+ Expr::Unnest(Unnest { expr, outer }) => expr.map_elements(f)?- .update_data(|expr| Expr::Unnest(Unnest { expr })),+ .update_data(|expr| Expr::Unnest(Unnest { expr, outer })),Expr::Alias(Alias {expr,relation,diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex fbe8492299..3865ff1d96 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -306,8 +306,19 @@ impl ExecutionPlan for UnnestExec {.iter().map(|index| *index as _).collect();+ let null_handling = {+ use datafusion_common::NullHandling;+ use protobuf::unnest_options::NullHandling as ProtoNullHandling;+ match self.options().null_handling {+ NullHandling::Preserve => ProtoNullHandling::Preserve,+ NullHandling::Drop => ProtoNullHandling::Drop,+ NullHandling::PreserveAndExpandEmpty => {+ ProtoNullHandling::PreserveAndExpandEmpty+ }+ }+ } as i32;let options = protobuf::UnnestOptions {- preserve_nulls: self.options().preserve_nulls,+ null_handling,recursions: self.options().recursions@@ -383,8 +394,22 @@ impl UnnestExec {"UnnestExec is missing required field 'options'")})?;+ let null_handling = {+ use datafusion_common::NullHandling;+ use protobuf::unnest_options::NullHandling as ProtoNullHandling;+ match ProtoNullHandling::try_from(options.null_handling) {+ Ok(ProtoNullHandling::Preserve) => NullHandling::Preserve,+ Ok(ProtoNullHandling::Drop) => NullHandling::Drop,+ Ok(ProtoNullHandling::PreserveAndExpandEmpty) => {+ NullHandling::PreserveAndExpandEmpty+ }+ // Unknown enum values fall back to the default (Preserve),+ // matching DataFusion's historical behavior.+ Err(_) => NullHandling::Preserve,+ }+ };let options = UnnestOptions {- preserve_nulls: options.preserve_nulls,+ null_handling,recursions: options.recursions.iter()@@ -888,14 +913,21 @@ fn build_batch(/// l2: [4,5], [], null, [6, 7]/// ```///-/// If `preserve_nulls` is false, the longest length array will be:+/// With [`datafusion_common::NullHandling::Drop`], the longest length array will be:////// ```ignore/// longest_length: [3, 0, 0, 2]/// ```///-/// whereas if `preserve_nulls` is true, the longest length array will be:+/// With [`datafusion_common::NullHandling::Preserve`] (the default), the longest length array+/// will be:+///+/// ```ignore+/// longest_length: [3, 1, 1, 2]+/// ```///+/// With [`datafusion_common::NullHandling::PreserveAndExpandEmpty`], empty input lists are+/// also bumped to length 1 so they produce a single `NULL` output row:////// ```ignore/// longest_length: [3, 1, 1, 2]@@ -904,12 +936,16 @@ fn find_longest_length(list_arrays: &[ArrayRef],options: &UnnestOptions,) -> Result<ArrayRef> {- // The length of a NULL list- let null_length = if options.preserve_nulls {+ // The length to substitute for a NULL input list.+ let null_length = if options.preserve_nulls() {Scalar::new(Int64Array::from_value(1, 1))} else {Scalar::new(Int64Array::from_value(0, 1))};+ let expand_empty = options.expand_empty_as_null();+ // Reused scalars for the empty-list rewrite when expand_empty is set.+ let zero = Scalar::new(Int64Array::from_value(0, 1));+ let one = Scalar::new(Int64Array::from_value(1, 1));let list_lengths: Vec<ArrayRef> = list_arrays.iter().map(|list_array| {@@ -918,6 +954,12 @@ fn find_longest_length(length_array = cast(&length_array, &DataType::Int64)?;length_array =zip(&is_not_null(&length_array)?, &length_array, &null_length)?;+ if expand_empty {+ // Bump empty lists (length 0) to length 1 so they+ // produce a single output row padded with NULL.+ let is_zero = arrow_ord::cmp::eq(&length_array, &zero)?;+ length_array = zip(&is_zero, &one, &length_array)?;+ }Ok(length_array)}).collect::<Result<_>>()?;@@ -1187,6 +1229,7 @@ mod tests {};use arrow::buffer::{NullBuffer, OffsetBuffer};use arrow::datatypes::{Field, Int32Type};+ use datafusion_common::NullHandling;use datafusion_common::test_util::batches_to_string;use insta::assert_snapshot;@@ -1369,12 +1412,375 @@ mod tests {list_type_columns.as_ref(),&HashSet::default(),&UnnestOptions {- preserve_nulls: true,+ null_handling: NullHandling::Preserve,+ recursions: vec![],+ },+ )?+ .unwrap();++ assert_snapshot!(batches_to_string(&[ret]),+ @r"+ +---------------------------------+---------------------------------+---------------------------------++ | col1_unnest_placeholder_depth_1 | col1_unnest_placeholder_depth_2 | col2_unnest_placeholder_depth_1 |+ +---------------------------------+---------------------------------+---------------------------------++ | [1, 2, 3] | 1 | a |+ | | 2 | b |+ | [4, 5] | 3 | |+ | [1, 2, 3] | | a |+ | | | b |+ | [4, 5] | | |+ | [1, 2, 3] | 4 | a |+ | | 5 | b |+ | [4, 5] | | |+ | [7, 8, 9, 10] | 7 | c |+ | | 8 | d |+ | [11, 12, 13] | 9 | |+ | | 10 | |+ | [7, 8, 9, 10] | | c |+ | | | d |+ | [11, 12, 13] | | |+ | [7, 8, 9, 10] | 11 | c |+ | | 12 | d |+ | [11, 12, 13] | 13 | |+ | | | e |+ +---------------------------------+---------------------------------+---------------------------------++ ");+ Ok(())+ }++ #[test]+ fn test_build_batch_preserve_and_expand_empty() -> Result<()> {+ // c1: [A, B, C], [], NULL, [D], NULL, [NULL, F] c2: 1, 2, 3, 4, 5, 6+ // Expected for `NullHandling::PreserveAndExpandEmpty`:+ // [A, B, C] -> three rows with c2 = 1, 1, 1+ // [] -> one row with c2 = 2 and unnested value NULL+ // NULL -> one row with c2 = 3 and unnested value NULL+ // [D] -> one row with c2 = 4+ // NULL -> one row with c2 = 5 and unnested value NULL+ // [NULL, F] -> two rows with c2 = 6, 6+ let list_array = Arc::new(make_generic_array::<i32>()) as ArrayRef;+ let other =+ Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3, 4, 5, 6])) as ArrayRef;+ let in_schema = Arc::new(Schema::new(vec![+ Field::new(+ "c1",+ DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))),+ true,+ ),+ Field::new("c2", DataType::Int32, true),+ ]));+ let out_schema = Arc::new(Schema::new(vec![+ Field::new("c1_unnested", DataType::Utf8, true),+ Field::new("c2", DataType::Int32, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&in_schema),+ vec![Arc::clone(&list_array), Arc::clone(&other)],+ )?;+ let list_type_columns = vec![ListUnnest {+ index_in_input_schema: 0,+ depth: 1,+ }];++ let ret = build_batch(+ &batch,+ &out_schema,+ &list_type_columns,+ &HashSet::default(),+ &UnnestOptions {+ null_handling: NullHandling::PreserveAndExpandEmpty,+ recursions: vec![],+ },+ )?+ .unwrap();++ assert_snapshot!(batches_to_string(&[ret]),+ @r"+ +-------------+----++ | c1_unnested | c2 |+ +-------------+----++ | A | 1 |+ | B | 1 |+ | C | 1 |+ | | 2 |+ | | 3 |+ | D | 4 |+ | | 5 |+ | | 6 |+ | F | 6 |+ +-------------+----++ ");+ Ok(())+ }++ // PreserveAndExpandEmpty must work for LargeListArray (i64 offsets) too,+ // not just the i32-offset ListArray exercised above.+ #[test]+ fn test_build_batch_preserve_and_expand_empty_largelist() -> Result<()> {+ let list_array = Arc::new(make_generic_array::<i64>()) as ArrayRef;+ let other =+ Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3, 4, 5, 6])) as ArrayRef;+ let in_schema = Arc::new(Schema::new(vec![+ Field::new(+ "c1",+ DataType::LargeList(Arc::new(Field::new_list_field(+ DataType::Utf8,+ true,+ ))),+ true,+ ),+ Field::new("c2", DataType::Int32, true),+ ]));+ let out_schema = Arc::new(Schema::new(vec![+ Field::new("c1_unnested", DataType::Utf8, true),+ Field::new("c2", DataType::Int32, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&in_schema),+ vec![Arc::clone(&list_array), Arc::clone(&other)],+ )?;+ let list_type_columns = vec![ListUnnest {+ index_in_input_schema: 0,+ depth: 1,+ }];++ let ret = build_batch(+ &batch,+ &out_schema,+ &list_type_columns,+ &HashSet::default(),+ &UnnestOptions {+ null_handling: NullHandling::PreserveAndExpandEmpty,+ recursions: vec![],+ },+ )?+ .unwrap();++ // Same expected shape as the ListArray case — exercises the LargeList+ // code path in unnest_list_array.+ assert_snapshot!(batches_to_string(&[ret]),+ @r"+ +-------------+----++ | c1_unnested | c2 |+ +-------------+----++ | A | 1 |+ | B | 1 |+ | C | 1 |+ | | 2 |+ | | 3 |+ | D | 4 |+ | | 5 |+ | | 6 |+ | F | 6 |+ +-------------+----++ ");+ Ok(())+ }++ // When two list columns are unnested together, `find_longest_length`+ // takes the per-row max. PreserveAndExpandEmpty must bump zeros to ones+ // in each input column independently, then the row-wise max picks up+ // the right value.+ #[test]+ fn test_build_batch_preserve_and_expand_empty_multi_column() -> Result<()> {+ // col_a: [1, 2], [], NULL, [3]+ // col_b: ['x'], ['y'],['z'], NULL+ let col_a = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2)]),+ Some(vec![]),+ None,+ Some(vec![Some(3)]),+ ]);+ let col_b = {+ let mut b =+ arrow::array::ListBuilder::new(arrow::array::StringBuilder::new());+ b.values().append_value("x");+ b.append(true);+ b.values().append_value("y");+ b.append(true);+ b.values().append_value("z");+ b.append(true);+ b.append(false);+ b.finish()+ };+ let id =+ Arc::new(arrow::array::Int32Array::from(vec![10, 20, 30, 40])) as ArrayRef;++ let in_schema = Arc::new(Schema::new(vec![+ Field::new(+ "a",+ DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),+ true,+ ),+ Field::new(+ "b",+ DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))),+ true,+ ),+ Field::new("id", DataType::Int32, true),+ ]));+ let out_schema = Arc::new(Schema::new(vec![+ Field::new("a_unnested", DataType::Int32, true),+ Field::new("b_unnested", DataType::Utf8, true),+ Field::new("id", DataType::Int32, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&in_schema),+ vec![+ Arc::new(col_a) as ArrayRef,+ Arc::new(col_b) as ArrayRef,+ Arc::clone(&id),+ ],+ )?;+ let list_type_columns = vec![+ ListUnnest {+ index_in_input_schema: 0,+ depth: 1,+ },+ ListUnnest {+ index_in_input_schema: 1,+ depth: 1,+ },+ ];++ let ret = build_batch(+ &batch,+ &out_schema,+ &list_type_columns,+ &HashSet::default(),+ &UnnestOptions {+ null_handling: NullHandling::PreserveAndExpandEmpty,recursions: vec![],},)?.unwrap();+ // Row 0: longest = max(len([1,2])=2, len(['x'])=1) = 2 → a=[1,2], b=['x',NULL]+ // Row 1: a=[] bumped to len 1, b=['y'] len 1 → a=[NULL], b=['y']+ // Row 2: a=NULL bumped to len 1, b=['z'] len 1 → a=[NULL], b=['z']+ // Row 3: a=[3] len 1, b=NULL bumped to len 1 → a=[3], b=[NULL]+ assert_snapshot!(batches_to_string(&[ret]),+ @r"+ +------------+------------+----++ | a_unnested | b_unnested | id |+ +------------+------------+----++ | 1 | x | 10 |+ | 2 | | 10 |+ | | y | 20 |+ | | z | 30 |+ | 3 | | 40 |+ +------------+------------+----++ ");+ Ok(())+ }++ // PreserveAndExpandEmpty must propagate through recursive depth-2+ // unnesting: an outer NULL or empty produces one NULL output row at+ // each level. Adapted from `test_build_batch_list_arr_recursive`.+ #[test]+ fn test_build_batch_preserve_and_expand_empty_recursive() -> Result<()> {+ // col1 | col2+ // [[1,2,3],null,[4,5]] | ['a','b']+ // [[7,8,9,10], null, [11,12,13]] | ['c','d']+ // null | ['e']+ let list_arr1 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ None,+ Some(vec![Some(4), Some(5)]),+ Some(vec![Some(7), Some(8), Some(9), Some(10)]),+ None,+ Some(vec![Some(11), Some(12), Some(13)]),+ ]);+ let list_arr1_ref = Arc::new(list_arr1) as ArrayRef;+ let offsets = OffsetBuffer::from_lengths([3, 3, 0]);+ let mut nulls = NullBufferBuilder::new(3);+ nulls.append_non_null();+ nulls.append_non_null();+ nulls.append_null();+ let col1_field = Field::new_list_field(+ DataType::List(Arc::new(Field::new_list_field(+ list_arr1_ref.data_type().to_owned(),+ true,+ ))),+ true,+ );+ let col1 = ListArray::new(+ Arc::new(Field::new_list_field(+ list_arr1_ref.data_type().to_owned(),+ true,+ )),+ offsets,+ list_arr1_ref,+ nulls.finish(),+ );++ let list_arr2 = StringArray::from(vec![+ Some("a"),+ Some("b"),+ Some("c"),+ Some("d"),+ Some("e"),+ ]);+ let offsets = OffsetBuffer::from_lengths([2, 2, 1]);+ let mut nulls = NullBufferBuilder::new(3);+ nulls.append_n_non_nulls(3);+ let col2_field = Field::new(+ "col2",+ DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))),+ true,+ );+ let col2 = GenericListArray::<i32>::new(+ Arc::new(Field::new_list_field(DataType::Utf8, true)),+ OffsetBuffer::new(offsets.into()),+ Arc::new(list_arr2),+ nulls.finish(),+ );+ let schema = Arc::new(Schema::new(vec![col1_field, col2_field]));+ let out_schema = Arc::new(Schema::new(vec![+ Field::new(+ "col1_unnest_placeholder_depth_1",+ DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),+ true,+ ),+ Field::new("col1_unnest_placeholder_depth_2", DataType::Int32, true),+ Field::new("col2_unnest_placeholder_depth_1", DataType::Utf8, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(col1) as ArrayRef, Arc::new(col2) as ArrayRef],+ )?;+ let list_type_columns = vec![+ ListUnnest {+ index_in_input_schema: 0,+ depth: 1,+ },+ ListUnnest {+ index_in_input_schema: 0,+ depth: 2,+ },+ ListUnnest {+ index_in_input_schema: 1,+ depth: 1,+ },+ ];++ let ret = build_batch(+ &batch,+ &out_schema,+ &list_type_columns,+ &HashSet::default(),+ &UnnestOptions {+ null_handling: NullHandling::PreserveAndExpandEmpty,+ recursions: vec![],+ },+ )?+ .unwrap();++ // The third input row (col1 = null, col2 = ['e']) now produces a+ // NULL row for the depth-1 col1 placeholder *and* the depth-2 one,+ // instead of being dropped at depth 1 and again at depth 2 the way+ // it would be under `Drop`. Inner NULLs inside [...null...] sub-+ // lists are still padded with NULL as before.assert_snapshot!(batches_to_string(&[ret]),@r"+---------------------------------+---------------------------------+---------------------------------+@@ -1452,11 +1858,11 @@ mod tests {fn verify_longest_length(list_arrays: &[ArrayRef],- preserve_nulls: bool,+ null_handling: NullHandling,expected: Vec<i64>,) -> Result<()> {let options = UnnestOptions {- preserve_nulls,+ null_handling,recursions: vec![],};let longest_length = find_longest_length(list_arrays, &options)?;@@ -1476,20 +1882,55 @@ mod tests {// Test with single ListArray// [A, B, C], [], NULL, [D], NULL, [NULL, F]let list_array = Arc::new(make_generic_array::<i32>()) as ArrayRef;- verify_longest_length(&[Arc::clone(&list_array)], false, vec![3, 0, 0, 1, 0, 2])?;- verify_longest_length(&[Arc::clone(&list_array)], true, vec![3, 0, 1, 1, 1, 2])?;+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::Drop,+ vec![3, 0, 0, 1, 0, 2],+ )?;+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::Preserve,+ vec![3, 0, 1, 1, 1, 2],+ )?;+ // PreserveAndExpandEmpty also treats empty lists as a NULL row.+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::PreserveAndExpandEmpty,+ vec![3, 1, 1, 1, 1, 2],+ )?;// Test with single LargeListArray// [A, B, C], [], NULL, [D], NULL, [NULL, F]let list_array = Arc::new(make_generic_array::<i64>()) as ArrayRef;- verify_longest_length(&[Arc::clone(&list_array)], false, vec![3, 0, 0, 1, 0, 2])?;- verify_longest_length(&[Arc::clone(&list_array)], true, vec![3, 0, 1, 1, 1, 2])?;+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::Drop,+ vec![3, 0, 0, 1, 0, 2],+ )?;+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::Preserve,+ vec![3, 0, 1, 1, 1, 2],+ )?;+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::PreserveAndExpandEmpty,+ vec![3, 1, 1, 1, 1, 2],+ )?;// Test with single FixedSizeListArray// [A, B], NULL, [C, D], NULL, [NULL, F], [NULL, NULL]let list_array = Arc::new(make_fixed_list()) as ArrayRef;- verify_longest_length(&[Arc::clone(&list_array)], false, vec![2, 0, 2, 0, 2, 2])?;- verify_longest_length(&[Arc::clone(&list_array)], true, vec![2, 1, 2, 1, 2, 2])?;+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::Drop,+ vec![2, 0, 2, 0, 2, 2],+ )?;+ verify_longest_length(+ &[Arc::clone(&list_array)],+ NullHandling::Preserve,+ vec![2, 1, 2, 1, 2, 2],+ )?;// Test with multiple list arrays// [A, B, C], [], NULL, [D], NULL, [NULL, F]@@ -1497,8 +1938,17 @@ mod tests {let list1 = Arc::new(make_generic_array::<i32>()) as ArrayRef;let list2 = Arc::new(make_fixed_list()) as ArrayRef;let list_arrays = vec![Arc::clone(&list1), Arc::clone(&list2)];- verify_longest_length(&list_arrays, false, vec![3, 0, 2, 1, 2, 2])?;- verify_longest_length(&list_arrays, true, vec![3, 1, 2, 1, 2, 2])?;+ verify_longest_length(&list_arrays, NullHandling::Drop, vec![3, 0, 2, 1, 2, 2])?;+ verify_longest_length(+ &list_arrays,+ NullHandling::Preserve,+ vec![3, 1, 2, 1, 2, 2],+ )?;+ verify_longest_length(+ &list_arrays,+ NullHandling::PreserveAndExpandEmpty,+ vec![3, 1, 2, 1, 2, 2],+ )?;Ok(())}diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 16b1b1532f..cbc41a7c57 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -403,7 +403,22 @@ message ColumnUnnestListRecursion {}message UnnestOptions {- bool preserve_nulls = 1;+ // Reserved for the historical `bool preserve_nulls = 1;` field.+ // Use `null_handling` instead.+ reserved 1;+ reserved "preserve_nulls";++ enum NullHandling {+ // Preserve nulls; empty lists produce no rows. The historical default.+ PRESERVE = 0;+ // Drop both null and empty lists from the output.+ DROP = 1;+ // Preserve nulls, and additionally expand empty lists into a single+ // NULL output row (outer-unnest semantics).+ PRESERVE_AND_EXPAND_EMPTY = 2;+ }++ NullHandling null_handling = 3;repeated RecursionUnnestOption recursions = 2;}@@ -618,6 +633,9 @@ message NegativeNode {message Unnest {repeated LogicalExprNode exprs = 1;+ // When true, this Unnest expression has outer-unnest semantics: NULL and+ // empty input lists both produce a single NULL output row.+ bool outer = 2;}message InListNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex c5d7c00301..7f9b9eddc5 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -26807,10 +26807,16 @@ impl serde::Serialize for Unnest {if !self.exprs.is_empty() {len += 1;}+ if self.outer {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.Unnest", len)?;if !self.exprs.is_empty() {struct_ser.serialize_field("exprs", &self.exprs)?;}+ if self.outer {+ struct_ser.serialize_field("outer", &self.outer)?;+ }struct_ser.end()}}@@ -26822,11 +26828,13 @@ impl<'de> serde::Deserialize<'de> for Unnest {{const FIELDS: &[&str] = &["exprs",+ "outer",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Exprs,+ Outer,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -26849,6 +26857,7 @@ impl<'de> serde::Deserialize<'de> for Unnest {{match value {"exprs" => Ok(GeneratedField::Exprs),+ "outer" => Ok(GeneratedField::Outer),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -26869,6 +26878,7 @@ impl<'de> serde::Deserialize<'de> for Unnest {V: serde::de::MapAccess<'de>,{let mut exprs__ = None;+ let mut outer__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Exprs => {@@ -26877,10 +26887,17 @@ impl<'de> serde::Deserialize<'de> for Unnest {}exprs__ = Some(map_.next_value()?);}+ GeneratedField::Outer => {+ if outer__.is_some() {+ return Err(serde::de::Error::duplicate_field("outer"));+ }+ outer__ = Some(map_.next_value()?);+ }}}Ok(Unnest {exprs: exprs__.unwrap_or_default(),+ outer: outer__.unwrap_or_default(),})}}@@ -27262,15 +27279,17 @@ impl serde::Serialize for UnnestOptions {{use serde::ser::SerializeStruct;let mut len = 0;- if self.preserve_nulls {+ if self.null_handling != 0 {len += 1;}if !self.recursions.is_empty() {len += 1;}let mut struct_ser = serializer.serialize_struct("datafusion.UnnestOptions", len)?;- if self.preserve_nulls {- struct_ser.serialize_field("preserveNulls", &self.preserve_nulls)?;+ if self.null_handling != 0 {+ let v = unnest_options::NullHandling::try_from(self.null_handling)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.null_handling)))?;+ struct_ser.serialize_field("nullHandling", &v)?;}if !self.recursions.is_empty() {struct_ser.serialize_field("recursions", &self.recursions)?;@@ -27285,14 +27304,14 @@ impl<'de> serde::Deserialize<'de> for UnnestOptions {D: serde::Deserializer<'de>,{const FIELDS: &[&str] = &[- "preserve_nulls",- "preserveNulls",+ "null_handling",+ "nullHandling","recursions",];#[allow(clippy::enum_variant_names)]enum GeneratedField {- PreserveNulls,+ NullHandling,Recursions,}impl<'de> serde::Deserialize<'de> for GeneratedField {@@ -27315,7 +27334,7 @@ impl<'de> serde::Deserialize<'de> for UnnestOptions {E: serde::de::Error,{match value {- "preserveNulls" | "preserve_nulls" => Ok(GeneratedField::PreserveNulls),+ "nullHandling" | "null_handling" => Ok(GeneratedField::NullHandling),"recursions" => Ok(GeneratedField::Recursions),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}@@ -27336,15 +27355,15 @@ impl<'de> serde::Deserialize<'de> for UnnestOptions {whereV: serde::de::MapAccess<'de>,{- let mut preserve_nulls__ = None;+ let mut null_handling__ = None;let mut recursions__ = None;while let Some(k) = map_.next_key()? {match k {- GeneratedField::PreserveNulls => {- if preserve_nulls__.is_some() {- return Err(serde::de::Error::duplicate_field("preserveNulls"));+ GeneratedField::NullHandling => {+ if null_handling__.is_some() {+ return Err(serde::de::Error::duplicate_field("nullHandling"));}- preserve_nulls__ = Some(map_.next_value()?);+ null_handling__ = Some(map_.next_value::<unnest_options::NullHandling>()? as i32);}GeneratedField::Recursions => {if recursions__.is_some() {@@ -27355,7 +27374,7 @@ impl<'de> serde::Deserialize<'de> for UnnestOptions {}}Ok(UnnestOptions {- preserve_nulls: preserve_nulls__.unwrap_or_default(),+ null_handling: null_handling__.unwrap_or_default(),recursions: recursions__.unwrap_or_default(),})}@@ -27363,6 +27382,80 @@ impl<'de> serde::Deserialize<'de> for UnnestOptions {deserializer.deserialize_struct("datafusion.UnnestOptions", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for unnest_options::NullHandling {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let variant = match self {+ Self::Preserve => "PRESERVE",+ Self::Drop => "DROP",+ Self::PreserveAndExpandEmpty => "PRESERVE_AND_EXPAND_EMPTY",+ };+ serializer.serialize_str(variant)+ }+}+impl<'de> serde::Deserialize<'de> for unnest_options::NullHandling {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "PRESERVE",+ "DROP",+ "PRESERVE_AND_EXPAND_EMPTY",+ ];++ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = unnest_options::NullHandling;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)+ })+ }++ fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)+ })+ }++ fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ match value {+ "PRESERVE" => Ok(unnest_options::NullHandling::Preserve),+ "DROP" => Ok(unnest_options::NullHandling::Drop),+ "PRESERVE_AND_EXPAND_EMPTY" => Ok(unnest_options::NullHandling::PreserveAndExpandEmpty),+ _ => Err(serde::de::Error::unknown_variant(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_any(GeneratedVisitor)+ }+}impl serde::Serialize for ValuesNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 2300f7192f..f763348308 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -670,11 +670,57 @@ pub struct ColumnUnnestListRecursion {}#[derive(Clone, PartialEq, ::prost::Message)]pub struct UnnestOptions {- #[prost(bool, tag = "1")]- pub preserve_nulls: bool,+ #[prost(enumeration = "unnest_options::NullHandling", tag = "3")]+ pub null_handling: i32,#[prost(message, repeated, tag = "2")]pub recursions: ::prost::alloc::vec::Vec<RecursionUnnestOption>,}+/// Nested message and enum types in `UnnestOptions`.+pub mod unnest_options {+ #[derive(+ Clone,+ Copy,+ Debug,+ PartialEq,+ Eq,+ Hash,+ PartialOrd,+ Ord,+ ::prost::Enumeration+ )]+ #[repr(i32)]+ pub enum NullHandling {+ /// Preserve nulls; empty lists produce no rows. The historical default.+ Preserve = 0,+ /// Drop both null and empty lists from the output.+ Drop = 1,+ /// Preserve nulls, and additionally expand empty lists into a single+ /// NULL output row (outer-unnest semantics).+ PreserveAndExpandEmpty = 2,+ }+ impl NullHandling {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Preserve => "PRESERVE",+ Self::Drop => "DROP",+ Self::PreserveAndExpandEmpty => "PRESERVE_AND_EXPAND_EMPTY",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "PRESERVE" => Some(Self::Preserve),+ "DROP" => Some(Self::Drop),+ "PRESERVE_AND_EXPAND_EMPTY" => Some(Self::PreserveAndExpandEmpty),+ _ => None,+ }+ }+ }+}#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]pub struct RecursionUnnestOption {#[prost(message, optional, tag = "1")]@@ -963,6 +1009,10 @@ pub struct NegativeNode {pub struct Unnest {#[prost(message, repeated, tag = "1")]pub exprs: ::prost::alloc::vec::Vec<LogicalExprNode>,+ /// When true, this Unnest expression has outer-unnest semantics: NULL and+ /// empty input lists both produce a single NULL output row.+ #[prost(bool, tag = "2")]+ pub outer: bool,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct InListNode {diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rsindex 6d9a73e06f..00cc7f6a9d 100644--- a/datafusion/proto/src/logical_plan/from_proto.rs+++ b/datafusion/proto/src/logical_plan/from_proto.rs@@ -64,8 +64,20 @@ use super::{AsLogicalPlan, LogicalExtensionCodec};impl FromProto<&protobuf::UnnestOptions> for UnnestOptions {fn from_proto(opts: &protobuf::UnnestOptions) -> Self {+ use datafusion_common::NullHandling;+ use protobuf::unnest_options::NullHandling as ProtoNullHandling;+ let null_handling = match ProtoNullHandling::try_from(opts.null_handling) {+ Ok(ProtoNullHandling::Preserve) => NullHandling::Preserve,+ Ok(ProtoNullHandling::Drop) => NullHandling::Drop,+ Ok(ProtoNullHandling::PreserveAndExpandEmpty) => {+ NullHandling::PreserveAndExpandEmpty+ }+ // Unknown enum values fall back to the default (Preserve), which+ // matches DataFusion's historical behavior.+ Err(_) => NullHandling::Preserve,+ };Self {- preserve_nulls: opts.preserve_nulls,+ null_handling,recursions: opts.recursions.iter()@@ -681,7 +693,10 @@ pub fn parse_expr(if exprs.len() != 1 {return Err(proto_error("Unnest must have exactly one expression"));}- Ok(Expr::Unnest(Unnest::new(exprs.swap_remove(0))))+ Ok(Expr::Unnest(Unnest {+ expr: Box::new(exprs.swap_remove(0)),+ outer: unnest.outer,+ }))}ExprType::InList(in_list) => Ok(Expr::InList(InList::new(Box::new(parse_required_expr(diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rsindex 23ce254e99..89de342ff0 100644--- a/datafusion/proto/src/logical_plan/to_proto.rs+++ b/datafusion/proto/src/logical_plan/to_proto.rs@@ -57,8 +57,17 @@ use crate::protobuf::LogicalPlanNode;impl FromProto<&UnnestOptions> for protobuf::UnnestOptions {fn from_proto(opts: &UnnestOptions) -> Self {+ use datafusion_common::NullHandling;+ use protobuf::unnest_options::NullHandling as ProtoNullHandling;+ let null_handling = match opts.null_handling {+ NullHandling::Preserve => ProtoNullHandling::Preserve,+ NullHandling::Drop => ProtoNullHandling::Drop,+ NullHandling::PreserveAndExpandEmpty => {+ ProtoNullHandling::PreserveAndExpandEmpty+ }+ } as i32;Self {- preserve_nulls: opts.preserve_nulls,+ null_handling,recursions: opts.recursions.iter()@@ -571,9 +580,10 @@ pub fn serialize_expr(expr_type: Some(ExprType::Negative(expr)),}}- Expr::Unnest(Unnest { expr }) => {+ Expr::Unnest(Unnest { expr, outer }) => {let expr = protobuf::Unnest {exprs: vec![serialize_expr(expr.as_ref(), codec)?],+ outer: *outer,};protobuf::LogicalExprNode {expr_type: Some(ExprType::Unnest(expr)),diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex 74f7253386..0e77aa76f4 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -2736,6 +2736,18 @@ fn roundtrip_inlist() {fn roundtrip_unnest() {let test_expr = Expr::Unnest(Unnest {expr: Box::new(col("col")),+ outer: false,+ });++ let ctx = SessionContext::new();+ roundtrip_expr_test(test_expr, ctx);+}++#[test]+fn roundtrip_unnest_outer() {+ let test_expr = Expr::Unnest(Unnest {+ expr: Box::new(col("col")),+ outer: true,});let ctx = SessionContext::new();diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 508ba5c020..19a5ca337d 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -2499,7 +2499,7 @@ fn roundtrip_unnest() -> Result<()> {Arc::new(Schema::new(vec![fa, fb0, fc1, fc2, fd0, fe1, fe2, fe3]));let input = Arc::new(EmptyExec::new(input_schema));let options = UnnestOptions {- preserve_nulls: false,+ null_handling: datafusion_common::NullHandling::Drop,recursions: vec![datafusion_common::RecursionUnnestOption {input_column: datafusion_common::Column::new_unqualified("b"),output_column: datafusion_common::Column::new_unqualified("b"),diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rsindex 701485eee7..e6bee31fbf 100644--- a/datafusion/sql/src/expr/function.rs+++ b/datafusion/sql/src/expr/function.rs@@ -546,15 +546,25 @@ impl<S: ContextProvider> SqlToRel<'_, S> {}}- // Build Unnest expression- if name.eq("unnest") {+ // Build Unnest expression.+ //+ // `unnest(col)` drops `NULL` and empty input lists (default SQL+ // semantics, matching DuckDB/PostgreSQL). `unnest_outer(col)` sets+ // `outer = true` so the downstream planner picks+ // `NullHandling::PreserveAndExpandEmpty`, which preserves `NULL`+ // and empty input lists as a single `NULL` output row.+ if name.eq("unnest") || name.eq("unnest_outer") {+ let outer = name.eq("unnest_outer");let mut exprs = self.function_args_to_expr(args, schema, planner_context)?;if exprs.len() != 1 {- return plan_err!("unnest() requires exactly one argument");+ return plan_err!("{name}() requires exactly one argument");}let expr = exprs.swap_remove(0);Self::check_unnest_arg(&expr, schema)?;- return Ok(Expr::Unnest(Unnest::new(expr)));+ return Ok(Expr::Unnest(Unnest {+ expr: Box::new(expr),+ outer,+ }));}if !order_by.is_empty() && is_function_window {diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex ba7353c424..bdab013144 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -33,9 +33,10 @@ use arrow::datatypes::DataType;use datafusion_common::error::DataFusionErrorBuilder;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion_common::{Column, DFSchema, DFSchemaRef, Result, not_impl_err, plan_err};-use datafusion_common::{RecursionUnnestOption, UnnestOptions};+use datafusion_common::{NullHandling, RecursionUnnestOption, UnnestOptions};use datafusion_expr::ExprSchemable;use datafusion_expr::builder::get_struct_unnested_columns;+use datafusion_expr::expr::Unnest as UnnestExpr;use datafusion_expr::expr::{PlannedReplaceSelectItem, WildcardOptions};use datafusion_expr::expr_rewriter::{normalize_col, normalize_col_with_schemas_and_ambiguity_check, normalize_sorts,@@ -665,8 +666,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {});}- // Set preserve_nulls to false to ensure compatibility with DuckDB and PostgreSQL- let mut unnest_options = UnnestOptions::new().with_preserve_nulls(false);+ // The default SQL `UNNEST` matches DuckDB/PostgreSQL: drop both+ // NULL and empty input lists. Outer-unnest (modelled as+ // `Unnest { outer: true }`) overrides that and selects+ // `NullHandling::PreserveAndExpandEmpty`. Mixing the two in a+ // single SELECT is a planning error because `UnnestOptions` is+ // per-`UnnestExec`, not per-column.+ let null_handling = collect_unnest_null_handling(&intermediate_expr_groups)?;+ let mut unnest_options =+ UnnestOptions::new().with_null_handling(null_handling);let mut unnest_col_vec = vec![];for (col, maybe_list_unnest) in unnest_columns.into_iter() {@@ -1451,3 +1459,45 @@ fn has_unnest_expr_recursively(expr: &Expr) -> bool {});has_unnest}++/// Walk `select_exprs`, observe every [`Expr::Unnest`] inside them, and+/// derive the [`NullHandling`] mode for the resulting [`UnnestOptions`].+///+/// * No unnest with `outer = true` → [`NullHandling::Drop`] (default SQL+/// `UNNEST(...)` semantics, matching DuckDB/PostgreSQL).+/// * Every unnest with `outer = true` → [`NullHandling::PreserveAndExpandEmpty`]+/// (outer-unnest semantics: `NULL` and empty input lists each produce a+/// single `NULL` output row).+/// * A mix of `outer = true` and `outer = false` in one SELECT → planning+/// error, because `UnnestOptions` applies per `Unnest` plan node, not+/// per output column.+fn collect_unnest_null_handling(expr_groups: &[Vec<Expr>]) -> Result<NullHandling> {+ let mut saw_outer = false;+ let mut saw_inner = false;+ for group in expr_groups {+ for expr in group {+ expr.apply(|e| {+ if let Expr::Unnest(UnnestExpr { outer, .. }) = e {+ if *outer {+ saw_outer = true;+ } else {+ saw_inner = true;+ }+ }+ Ok(TreeNodeRecursion::Continue)+ })?;+ }+ }+ if saw_outer && saw_inner {+ return plan_err!(+ "Cannot mix `unnest(...)` with `unnest_outer(...)` in the same \+ SELECT — the unnest operator carries a single null-handling \+ mode. Split the query so each unnest projection uses one mode."+ );+ }+ Ok(if saw_outer {+ NullHandling::PreserveAndExpandEmpty+ } else {+ NullHandling::Drop+ })+}diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 89560b2379..9403e15406 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -2452,6 +2452,7 @@ mod tests {name: "array_col".to_string(),spans: Spans::new(),})),+ outer: false,}),r#"UNNEST("table".array_col)"#,),diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex 9fe97a8291..f4b60176cf 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -2047,7 +2047,7 @@ impl Unparser<'_> {let mut flatten = FlattenRelationBuilder::default();flatten.input_expr(input_expr);- flatten.outer(unnest.options.preserve_nulls);+ flatten.outer(unnest.options.preserve_nulls());Ok(Some(flatten))}diff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.sltindex 5cca3cbfe4..a3385b81d7 100644--- a/datafusion/sqllogictest/test_files/unnest.slt+++ b/datafusion/sqllogictest/test_files/unnest.slt@@ -1463,3 +1463,186 @@ FROM list_struct_table;statement okDROP TABLE list_struct_table;++####################################+# `unnest_outer` Tests+#+# `unnest_outer(col)` is the outer-unnest peer to `unnest(col)`. Rows whose+# input list is `NULL` or empty produce a single output row containing+# `NULL`; rows with values are exploded element-by-element the same way as+# plain `unnest`.+#+# Column types on the tables below are inferred from the `VALUES` rows.+# DataFusion's SQL parser does not accept PostgreSQL `TYPE[]` array-column+# syntax inside `CREATE TABLE ... AS VALUES`, so tables are declared as+# CTAS over a `VALUES` subquery with aliased column names.+####################################++## unnest vs unnest_outer on an integer list++statement ok+CREATE TABLE int_lists AS+SELECT column1 AS id, column2 AS xs FROM (VALUES+ (1, [10, 20, 30]),+ (4, [40]),+ (5, [NULL, 50]),+ (2, arrow_cast(make_array(), 'List(Int64)')),+ (3, NULL)+);++## Plain `unnest`: drops both NULL and empty input rows.+## Inner NULL elements survive.+query II+SELECT id, unnest(xs) AS x FROM int_lists ORDER BY id, x;+----+1 10+1 20+1 30+4 40+5 50+5 NULL++## `unnest_outer`: NULL and empty input lists each produce one NULL row.+query II+SELECT id, unnest_outer(xs) AS x FROM int_lists ORDER BY id, x;+----+1 10+1 20+1 30+2 NULL+3 NULL+4 40+5 50+5 NULL++## String list with inner NULLs: inner NULL elements must survive (they are+## not the same as "empty"), while NULL and empty input lists become a+## single NULL output row.++statement ok+CREATE TABLE str_lists AS+SELECT column1 AS id, column2 AS tags FROM (VALUES+ ('A', ['x', 'y']),+ ('B', ['p', NULL, 'q']),+ ('C', arrow_cast(make_array(), 'List(Utf8)')),+ ('D', NULL)+);++query TT+SELECT id, unnest_outer(tags) AS tag FROM str_lists ORDER BY id, tag;+----+A x+A y+B p+B q+B NULL+C NULL+D NULL++## Mixed list lengths — verify row-wise expansion.++statement ok+CREATE TABLE varied_lists AS+SELECT column1 AS id, column2 AS xs FROM (VALUES+ (1, [1, 2, 3, 4]),+ (2, [5]),+ (3, arrow_cast(make_array(), 'List(Int64)')),+ (4, NULL)+);++query II+SELECT id, unnest_outer(xs) AS x FROM varied_lists ORDER BY id, x;+----+1 1+1 2+1 3+1 4+2 5+3 NULL+4 NULL++## Aliased output column.+query II+SELECT id, unnest_outer(xs) AS unwrapped FROM int_lists WHERE id = 2;+----+2 NULL++## Mixing `unnest` and `unnest_outer` in one SELECT is a planning error.+## `UnnestOptions` is per-`UnnestExec`, so we refuse to silently pick one mode.++statement error DataFusion error: Error during planning: Cannot mix `unnest\(\.\.\.\)` with `unnest_outer\(\.\.\.\)` in the same SELECT+SELECT unnest(xs), unnest_outer(xs) FROM int_lists;++## Chained `unnest` → `unnest_outer` via subquery.+## `unnest(xs)` (inner) drops NULL and empty outer rows, then+## `unnest_outer(ys)` (outer) preserves NULL and empty sub-lists from the+## inner unnest.++statement ok+CREATE TABLE nested_lists AS+SELECT column1 AS id, column2 AS xs FROM (VALUES+ (100, [[1, 2, 3], NULL, [4, 5]]),+ (200, [[7], arrow_cast(make_array(), 'List(Int64)')]),+ (300, NULL)+);++query II+SELECT id, unnest_outer(ys) AS y+FROM (SELECT id, unnest(xs) AS ys FROM nested_lists)+ORDER BY id, y;+----+100 1+100 2+100 3+100 4+100 5+100 NULL+200 7+200 NULL++statement ok+DROP TABLE nested_lists;++## `unnest_outer` agrees with `unnest` when no NULL or empty rows exist.++statement ok+CREATE TABLE dense_lists AS+SELECT column1 AS id, column2 AS xs FROM (VALUES+ (1, [10, 20]),+ (2, [30]),+ (3, [40, 50, 60])+);++query II+SELECT id, unnest(xs) AS x FROM dense_lists ORDER BY id, x;+----+1 10+1 20+2 30+3 40+3 50+3 60++query II+SELECT id, unnest_outer(xs) AS x FROM dense_lists ORDER BY id, x;+----+1 10+1 20+2 30+3 40+3 50+3 60++## Cleanup++statement ok+DROP TABLE int_lists;++statement ok+DROP TABLE str_lists;++statement ok+DROP TABLE varied_lists;++statement ok+DROP TABLE dense_lists;diff --git a/docs/source/user-guide/sql/select.md b/docs/source/user-guide/sql/select.mdindex ea96f6ae45..af442de659 100644--- a/docs/source/user-guide/sql/select.md+++ b/docs/source/user-guide/sql/select.md@@ -279,6 +279,29 @@ SELECT id, UNNEST(items) FROM orders;items (implicit lateral references such as `FROM orders AS t, UNNEST(t.items)`are not currently supported).+### `unnest_outer`++`unnest_outer(col)` is the outer-unnest peer to `UNNEST(col)`. The two differ+only in how `NULL` and empty input lists are handled:++| Form | `NULL` input list | Empty input list |+| ------------------- | ----------------- | ---------------- |+| `UNNEST(col)` | dropped | dropped |+| `unnest_outer(col)` | one `NULL` row | one `NULL` row |++```sql+SELECT id, unnest_outer(tags) AS tag FROM rows;+```++An input row with an empty `tags` array or `NULL` `tags` produces one output+row whose `tag` is `NULL`, instead of being dropped. This is analogous to the+outer variant offered by other engines (Spark `explode_outer`, Hive `EXPLODE OUTER`, Snowflake `FLATTEN(OUTER => true)`).++`unnest_outer` cannot be mixed with `unnest` in the same `SELECT` — the+unnest plan node carries a single null-handling mode for all its output+columns, so a mix would be ambiguous. The planner returns an error in that+case.+## WHERE clause```text