Apache DataFusion SQL Query Engine
fix: UNNEST now preserves item fields in expression inference and logical-plan schemas (#26433)
Ethan Tran79650ec0e27d
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex d011a25273..1a4f6ffcb4 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -4483,6 +4483,210 @@ async fn unnest_dict_encoded_columns() -> Result<()> {Ok(())}+#[rstest]+#[case::list(DataType::List)]+#[case::large_list(DataType::LargeList)]+#[case::fixed_size_list(|field| DataType::FixedSizeList(field, 2))]+#[case::list_view(DataType::ListView)]+#[case::large_list_view(DataType::LargeListView)]+#[tokio::test]+async fn unnest_list_item_metadata(+ #[case] list_type: fn(Arc<Field>) -> DataType,+) -> Result<()> {+ use arrow::array::BinaryArray;+ use arrow::buffer::OffsetBuffer;+ use arrow::compute::cast;+ use datafusion_common::RecursionUnnestOption;++ let item = Arc::new(Field::new("wkb", DataType::Binary, false).with_metadata(+ HashMap::from([+ ("ARROW:extension:name".into(), "geoarrow.wkb".into()),+ (+ "ARROW:extension:metadata".into(),+ r#"{"crs":"EPSG:4326"}"#.into(),+ ),+ ("custom".into(), "item".into()),+ ]),+ ));+ let values: ArrayRef =+ Arc::new(BinaryArray::from_iter_values([b"a", b"b", b"c", b"d"]));+ let lists = ListArray::new(+ Arc::clone(&item),+ OffsetBuffer::new(vec![0, 2, 4].into()),+ Arc::clone(&values),+ None,+ );+ let lists = cast(&lists, &list_type(Arc::clone(&item)))?;+ let nested_item = Arc::new(+ Field::new("lists", lists.data_type().clone(), false)+ .with_metadata(HashMap::from([("custom".into(), "nested item".into())])),+ );+ let nested = ListArray::new(+ Arc::clone(&nested_item),+ OffsetBuffer::new(vec![0, 2].into()),+ Arc::clone(&lists),+ None,+ );+ let nested = cast(&nested, &list_type(Arc::clone(&nested_item)))?;+ let schema = Arc::new(Schema::new(vec![+ Field::new("input", nested.data_type().clone(), false)+ .with_metadata(HashMap::from([("custom".into(), "container".into())])),+ ]));+ let ctx = SessionContext::new();+ let batch = RecordBatch::try_new(schema, vec![nested])?;+ ctx.register_batch("t", batch.clone())?;+ let df = ctx.read_batch(batch)?;++ // Each recursion depth must use that element's metadata, rather than the+ // outer list's metadata. Renaming the result must retain it as well.+ for (depth, field, expected_values) in+ [(1, &nested_item, &lists), (2, &item, &values)]+ {+ let options = UnnestOptions::new().with_recursions(RecursionUnnestOption {+ input_column: "input".into(),+ output_column: "expanded".into(),+ depth,+ });+ let unnested = df+ .clone()+ .unnest_columns_with_options(&["input"], options)?+ .select(vec![col("expanded").alias("output")])?;+ let expected_field = field+ .as_ref()+ .clone()+ .with_name("output")+ .with_nullable(true);+ let sql = if depth == 1 {+ "SELECT unnest(input) AS output FROM t"+ } else {+ "SELECT unnest(unnest(input)) AS output FROM t"+ };+ for unnested in [unnested, ctx.sql(sql).await?] {+ assert_eq!(unnested.schema().field(0).as_ref(), &expected_field);+ let batches = unnested.collect().await?;+ assert_eq!(batches.len(), 1);+ assert_eq!(batches[0].schema().field(0), &expected_field);+ assert_eq!(batches[0].column(0), expected_values);+ }+ }+ Ok(())+}++#[tokio::test]+async fn unnest_list_item_metadata_in_udf() -> Result<()> {+ use arrow::buffer::OffsetBuffer;+ use datafusion_expr::{+ ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,+ };++ #[derive(Debug, PartialEq, Eq, Hash)]+ struct RequireItemMetadata(Signature);++ impl ScalarUDFImpl for RequireItemMetadata {+ fn name(&self) -> &str {+ "require_item_metadata"+ }++ fn signature(&self) -> &Signature {+ &self.0+ }++ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {+ Ok(arg_types[0].clone())+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<Arc<Field>> {+ let field = &args.arg_fields[0];+ if field.metadata().get("custom").map(String::as_str) != Some("item") {+ return datafusion_common::plan_err!("Missing item metadata");+ }+ Ok(Arc::clone(field))+ }++ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ Ok(args.args[0].clone())+ }+ }++ let item = Arc::new(+ Field::new("item", DataType::Int32, false)+ .with_metadata(HashMap::from([("custom".into(), "item".into())])),+ );+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));+ let list = ListArray::new(+ Arc::clone(&item),+ OffsetBuffer::new(vec![0, 2].into()),+ Arc::clone(&values),+ None,+ );+ let schema = Arc::new(Schema::new(vec![Field::new(+ "input",+ list.data_type().clone(),+ false,+ )]));+ let ctx = SessionContext::new();+ ctx.register_batch("t", RecordBatch::try_new(schema, vec![Arc::new(list)])?)?;+ ctx.register_udf(ScalarUDF::from(RequireItemMetadata(Signature::any(+ 1,+ Volatility::Immutable,+ ))));+ let expected_field = item+ .as_ref()+ .clone()+ .with_name("output")+ .with_nullable(true);++ // A UDF can inspect argument metadata before SQL rewrites UNNEST into a plan.+ for sql in [+ "SELECT require_item_metadata(unnest(input)) AS output FROM t",+ "SELECT require_item_metadata(output) AS output \+ FROM (SELECT unnest(input) AS output FROM t)",+ ] {+ let df = ctx.sql(sql).await?;+ assert_eq!(df.schema().field(0).as_ref(), &expected_field);+ let batches = df.collect().await?;+ assert_eq!(batches.len(), 1);+ assert_eq!(batches[0].schema().field(0), &expected_field);+ assert_eq!(batches[0].column(0), &values);+ }+ Ok(())+}++#[tokio::test]+async fn unnest_list_item_metadata_with_nulls() -> Result<()> {+ let item = Arc::new(+ Field::new("item", DataType::Int32, false)+ .with_metadata(HashMap::from([("custom".into(), "item".into())])),+ );+ let mut builder = ListBuilder::new(Int32Builder::new()).with_field(Arc::clone(&item));+ builder.values().append_slice(&[1, 2]);+ builder.append(true);+ builder.append(true); // Empty list+ builder.append(false); // NULL list+ let list = builder.finish();+ let schema = Arc::new(Schema::new(vec![Field::new(+ "input",+ list.data_type().clone(),+ true,+ )]));+ let ctx = SessionContext::new();+ let options = UnnestOptions::new()+ .with_null_handling(datafusion_common::NullHandling::PreserveAndExpandEmpty);+ let df = ctx+ .read_batch(RecordBatch::try_new(schema, vec![Arc::new(list)])?)?+ .unnest_columns_with_options(&["input"], options)?;+ let expected_field = item.as_ref().clone().with_name("input").with_nullable(true);+ assert_eq!(df.schema().field(0).as_ref(), &expected_field);+ let batches = df.collect().await?;+ assert_eq!(batches.len(), 1);+ assert_eq!(batches[0].schema().field(0), &expected_field);+ assert_eq!(+ batches[0].column(0).as_ref(),+ &Int32Array::from(vec![Some(1), Some(2), None, None]) as &dyn Array+ );+ Ok(())+}+#[tokio::test]async fn unnest_column_nulls() -> Result<()> {let df = table_with_lists_and_nulls().await?;diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rsindex 8aaca2b72b..2804c736c4 100644--- a/datafusion/expr/src/expr_schema.rs+++ b/datafusion/expr/src/expr_schema.rs@@ -517,6 +517,7 @@ impl ExprSchemable for Expr {/// - **Column references**: Preserve original field metadata from input schema/// - **Literals**: Use explicitly provided metadata, otherwise empty/// - **Aliases**: Merge underlying expr metadata with alias-specific metadata, preferring the alias metadata+ /// - **List unnesting**: Preserve the element field metadata/// - **Binary expressions**: field metadata is empty/// - **Boolean expressions**: field metadata is empty/// - **Cast expressions**: Type-only casts pass through source metadata (stripping extension@@ -678,6 +679,27 @@ impl ExprSchemable for Expr {Expr::LambdaVariable(LambdaVariable {field: Some(field), ..}) => Ok(Arc::clone(field).renamed(&schema_name)),+ Expr::Unnest(Unnest { expr, .. }) => {+ let nullable = self.nullable(schema)?;+ match expr.get_type(schema)? {+ DataType::List(field)+ | DataType::LargeList(field)+ | DataType::FixedSizeList(field, _)+ | DataType::ListView(field)+ | DataType::LargeListView(field) => Ok(Arc::new(+ field+ .as_ref()+ .clone()+ .with_name(&schema_name)+ .with_nullable(nullable),+ )),+ _ => Ok(Arc::new(Field::new(+ &schema_name,+ self.get_type(schema)?,+ nullable,+ ))),+ }+ }Expr::Like(_)| Expr::SimilarTo(_)| Expr::Not(_)@@ -689,7 +711,6 @@ impl ExprSchemable for Expr {| Expr::Wildcard { .. }| Expr::GroupingSet(_)| Expr::Placeholder(_)- | Expr::Unnest(_)| Expr::Lambda(_)| Expr::LambdaVariable(_) => Ok(Arc::new(Field::new(&schema_name,diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex a2fee99ec2..10cc92d6b5 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -44,8 +44,8 @@ use crate::logical_plan::{DmlStatement, Statement, WriteOp};use crate::utils::{check_aggregate_and_window_nesting, check_no_window_functions,enumerate_grouping_sets, expr_to_columns, exprlist_to_fields,- find_out_reference_exprs, grouping_set_expr_count, grouping_set_to_exprlist,- merge_schema, passthrough_field_index, split_conjunction,+ find_out_reference_exprs, get_unnested_list_field, grouping_set_expr_count,+ grouping_set_to_exprlist, merge_schema, passthrough_field_index, split_conjunction,};use crate::{BinaryExpr, CreateMemoryTable, CreateView, Execute, Expr, ExprSchemable, GroupingSet,@@ -5338,13 +5338,15 @@ fn get_unnested_columns(| DataType::LargeList(_)| DataType::ListView(_)| DataType::LargeListView(_) => {- let data_type = get_unnested_list_datatype_recursive(data_type, depth)?;- let new_field = Arc::new(Field::new(- col_name, data_type,- // Unnesting may produce NULLs even if the list is not null.- // For example: unnest([1], []) -> 1, null- true,- ));+ let field = get_unnested_list_field(data_type, depth)?;+ let new_field = Arc::new(+ field+ .clone()+ .with_name(col_name)+ // Unnesting may produce NULLs even if the list is not null.+ // For example: unnest([1], []) -> 1, null+ .with_nullable(true),+ );let column = Column::from_name(col_name);// let column = Column::from((None, &new_field));qualified_columns.push((column, new_field));@@ -5369,29 +5371,6 @@ fn get_unnested_columns(Ok(qualified_columns)}-// Get the data type of a multi-dimensional type after unnesting it-// with a given depth-fn get_unnested_list_datatype_recursive(- data_type: &DataType,- depth: usize,-) -> Result<DataType> {- match data_type {- DataType::List(field)- | DataType::FixedSizeList(field, _)- | DataType::LargeList(field)- | DataType::ListView(field)- | DataType::LargeListView(field) => {- if depth == 1 {- return Ok(field.data_type().clone());- }- return get_unnested_list_datatype_recursive(field.data_type(), depth - 1);- }- _ => {}- }-- internal_err!("trying to unnest on invalid data type {data_type}")-}-#[cfg(test)]mod tests {use super::*;diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rsindex 777e835078..8b9d25e2fb 100644--- a/datafusion/expr/src/utils.rs+++ b/datafusion/expr/src/utils.rs@@ -919,6 +919,28 @@ whereerr}+/// Returns the element field at the given list unnesting depth.+pub(crate) fn get_unnested_list_field(+ data_type: &DataType,+ depth: usize,+) -> Result<&Field> {+ match data_type {+ DataType::List(field)+ | DataType::FixedSizeList(field, _)+ | DataType::LargeList(field)+ | DataType::ListView(field)+ | DataType::LargeListView(field) => {+ if depth == 1 {+ return Ok(field.as_ref());+ }+ return get_unnested_list_field(field.data_type(), depth - 1);+ }+ _ => {}+ }++ internal_err!("trying to unnest on invalid data type {data_type}")+}+/// Create schema fields from an expression list, for use in result set schema construction////// This function converts a list of expressions into a list of complete schema fields,