Apache DataFusion SQL Query Engine
Add end-to-end Parquet tests for List and LargeList struct schema evolution (#20840)
kosiew6412c3aaa0e7
diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rsindex f412cdf9bd..cf32efbd70 100644--- a/datafusion/core/tests/parquet/expr_adapter.rs+++ b/datafusion/core/tests/parquet/expr_adapter.rs@@ -18,9 +18,11 @@use std::sync::Arc;use arrow::array::{- Array, ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatch, StringArray,- StructArray, record_batch,+ Array, ArrayRef, BooleanArray, Int32Array, Int64Array, LargeListArray, ListArray,+ RecordBatch, StringArray, StructArray, record_batch,};+use arrow::buffer::OffsetBuffer;+use arrow::compute::concat_batches;use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion::assert_batches_eq;@@ -54,6 +56,405 @@ async fn write_parquet(batch: RecordBatch, store: Arc<dyn ObjectStore>, path: &sstore.put(&Path::from(path), data.into()).await.unwrap();}+#[derive(Debug, Clone, Copy)]+enum NestedListKind {+ List,+ LargeList,+}++impl NestedListKind {+ fn field_data_type(self, item_field: Arc<Field>) -> DataType {+ match self {+ Self::List => DataType::List(item_field),+ Self::LargeList => DataType::LargeList(item_field),+ }+ }++ fn array(+ self,+ item_field: Arc<Field>,+ lengths: Vec<usize>,+ values: ArrayRef,+ ) -> ArrayRef {+ match self {+ Self::List => Arc::new(ListArray::new(+ item_field,+ OffsetBuffer::<i32>::from_lengths(lengths),+ values,+ None,+ )),+ Self::LargeList => Arc::new(LargeListArray::new(+ item_field,+ OffsetBuffer::<i64>::from_lengths(lengths),+ values,+ None,+ )),+ }+ }++ fn name(self) -> &'static str {+ match self {+ Self::List => "list",+ Self::LargeList => "large_list",+ }+ }+}++#[derive(Debug)]+// Fixture row for one nested struct element inside the `messages` list column.+struct NestedMessageRow<'a> {+ id: i32,+ name: &'a str,+ chain: Option<&'a str>,+ ignored: Option<i32>,+}++fn message_fields(+ chain_type: DataType,+ chain_nullable: bool,+ include_chain: bool,+ include_ignored: bool,+) -> Fields {+ let mut fields = vec![+ Arc::new(Field::new("id", DataType::Int32, false)),+ Arc::new(Field::new("name", DataType::Utf8, true)),+ ];+ if include_chain {+ fields.push(Arc::new(Field::new("chain", chain_type, chain_nullable)));+ }+ if include_ignored {+ fields.push(Arc::new(Field::new("ignored", DataType::Int32, true)));+ }+ fields.into()+}++// Helper to construct the target message schema for struct evolution tests.+// The schema always has id (Int64), name (Utf8), and chain with parameterized type.+fn target_message_fields(chain_type: DataType, chain_nullable: bool) -> Fields {+ vec![+ Arc::new(Field::new("id", DataType::Int64, false)),+ Arc::new(Field::new("name", DataType::Utf8, true)),+ Arc::new(Field::new("chain", chain_type, chain_nullable)),+ ]+ .into()+}++// Helper to build message columns in canonical order (id, name, chain, ignored)+// based on which optional fields are present in the schema.+fn build_message_columns(+ id_array: &ArrayRef,+ name_array: &ArrayRef,+ chain_vec: &[Option<&str>],+ ignored_array: &ArrayRef,+ fields: &Fields,+) -> Vec<ArrayRef> {+ let mut columns = vec![Arc::clone(id_array), Arc::clone(name_array)];++ for field in fields.iter().skip(2) {+ match field.name().as_str() {+ "chain" => {+ let chain_array = match field.data_type() {+ DataType::Utf8 => {+ Arc::new(StringArray::from(chain_vec.to_vec())) as ArrayRef+ }+ DataType::Struct(chain_fields) => {+ let chain_struct = StructArray::new(+ chain_fields.clone(),+ vec![Arc::new(StringArray::from(chain_vec.to_vec()))+ as ArrayRef],+ None,+ );+ Arc::new(chain_struct) as ArrayRef+ }+ other => panic!("unexpected chain field type: {other:?}"),+ };+ columns.push(chain_array);+ }+ "ignored" => columns.push(Arc::clone(ignored_array)),+ _ => {}+ }+ }+ columns+}++fn nested_messages_batch(+ kind: NestedListKind,+ row_id: i32,+ messages: &[NestedMessageRow<'_>],+ fields: &Fields,+) -> RecordBatch {+ let item_field = Arc::new(Field::new("item", DataType::Struct(fields.clone()), true));++ let (ids_vec, names_vec, chain_vec, ignored_vec) = messages.iter().fold(+ (+ Vec::with_capacity(messages.len()),+ Vec::with_capacity(messages.len()),+ Vec::with_capacity(messages.len()),+ Vec::with_capacity(messages.len()),+ ),+ |(mut ids, mut names, mut chains, mut ignoreds), msg| {+ ids.push(msg.id);+ names.push(Some(msg.name));+ chains.push(msg.chain);+ ignoreds.push(msg.ignored);+ (ids, names, chains, ignoreds)+ },+ );++ // Build all arrays once+ let id_array = Arc::new(Int32Array::from(ids_vec)) as ArrayRef;+ let name_array = Arc::new(StringArray::from(names_vec)) as ArrayRef;+ let ignored_array = Arc::new(Int32Array::from(ignored_vec)) as ArrayRef;++ // Build columns in canonical order (id, name, chain, ignored) based on field schema+ let columns =+ build_message_columns(&id_array, &name_array, &chain_vec, &ignored_array, fields);++ let struct_array = StructArray::new(fields.clone(), columns, None);++ // Compute the message data type first, then move item_field into kind.array()+ let message_data_type = kind.field_data_type(item_field.clone());+ let messages_array =+ kind.array(item_field, vec![messages.len()], Arc::new(struct_array));+ let schema = Arc::new(Schema::new(vec![+ Field::new("row_id", DataType::Int32, false),+ Field::new("messages", message_data_type, true),+ ]));++ RecordBatch::try_new(+ schema,+ vec![+ Arc::new(Int32Array::from(vec![row_id])) as ArrayRef,+ messages_array,+ ],+ )+ .unwrap()+}++async fn register_memory_listing_table(+ ctx: &SessionContext,+ store: Arc<dyn ObjectStore>,+ base_path: &str,+ table_schema: SchemaRef,+) {+ let store_url = ObjectStoreUrl::parse("memory://").unwrap();+ ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));++ let listing_table_config =+ ListingTableConfig::new(ListingTableUrl::parse(base_path).unwrap())+ .infer_options(&ctx.state())+ .await+ .unwrap()+ .with_schema(table_schema)+ .with_expr_adapter_factory(Arc::new(DefaultPhysicalExprAdapterFactory));++ let table = ListingTable::try_new(listing_table_config).unwrap();+ ctx.register_table("t", Arc::new(table)).unwrap();+}++fn test_context() -> SessionContext {+ let mut cfg = SessionConfig::new()+ .with_collect_statistics(false)+ .with_parquet_pruning(false)+ .with_parquet_page_index_pruning(false);+ cfg.options_mut().execution.parquet.pushdown_filters = true;+ SessionContext::new_with_config(cfg)+}++fn nested_list_table_schema(+ kind: NestedListKind,+ target_message_fields: Fields,+) -> SchemaRef {+ let target_item = Arc::new(Field::new(+ "item",+ DataType::Struct(target_message_fields),+ true,+ ));+ Arc::new(Schema::new(vec![+ Field::new("row_id", DataType::Int32, false),+ Field::new("messages", kind.field_data_type(target_item), true),+ ]))+}++// Helper to extract message values from a nested list column.+// Returns the values at indices 0 and 1 from either a ListArray or LargeListArray.+fn extract_nested_list_values(+ kind: NestedListKind,+ column: &ArrayRef,+) -> (ArrayRef, ArrayRef) {+ match kind {+ NestedListKind::List => {+ let list = column+ .as_any()+ .downcast_ref::<ListArray>()+ .expect("messages should be a ListArray");+ (list.value(0), list.value(1))+ }+ NestedListKind::LargeList => {+ let list = column+ .as_any()+ .downcast_ref::<LargeListArray>()+ .expect("messages should be a LargeListArray");+ (list.value(0), list.value(1))+ }+ }+}++// Helper to set up a nested list test fixture.+// Creates an in-memory store, writes the provided batches to parquet files,+// creates a SessionContext, and registers the resulting table.+// Returns the prepared context ready for queries.+async fn setup_nested_list_test(+ kind: NestedListKind,+ prefix_base: &str,+ batches: Vec<(String, RecordBatch)>,+ table_schema: SchemaRef,+) -> SessionContext {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ let prefix = format!("{}_{}", kind.name(), prefix_base);++ for (filename, batch) in batches {+ write_parquet(batch, Arc::clone(&store), &format!("{prefix}/{filename}")).await;+ }++ let ctx = test_context();+ register_memory_listing_table(+ &ctx,+ Arc::clone(&store),+ &format!("memory:///{prefix}/"),+ table_schema,+ )+ .await;++ ctx+}++async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Result<()> {+ // old.parquet shape: messages item struct has only (id, name), no `chain`.+ let old_batch = nested_messages_batch(+ kind,+ 1,+ &[+ NestedMessageRow {+ id: 10,+ name: "alpha",+ chain: None,+ ignored: None,+ },+ NestedMessageRow {+ id: 20,+ name: "beta",+ chain: None,+ ignored: None,+ },+ ],+ &message_fields(DataType::Utf8, true, false, false),+ );++ // new.parquet shape: messages item struct adds nullable `chain` and extra `ignored`.+ let new_batch = nested_messages_batch(+ kind,+ 2,+ &[NestedMessageRow {+ id: 30,+ name: "gamma",+ chain: Some("eth"),+ ignored: Some(99),+ }],+ &message_fields(DataType::Utf8, true, true, true),+ );++ // Logical table schema expects evolved shape (id, name, nullable `chain`) and+ // should ignore source-only `ignored` during reads.+ let table_schema =+ nested_list_table_schema(kind, target_message_fields(DataType::Utf8, true));++ let ctx = setup_nested_list_test(+ kind,+ "struct_evolution",+ vec![+ ("old.parquet".to_string(), old_batch),+ ("new.parquet".to_string(), new_batch),+ ],+ table_schema,+ )+ .await;++ let select_all = ctx+ .sql("SELECT * FROM t ORDER BY row_id")+ .await?+ .collect()+ .await?;+ let all_rows = concat_batches(&select_all[0].schema(), &select_all)?;++ let row_ids = all_rows+ .column(0)+ .as_any()+ .downcast_ref::<Int32Array>()+ .expect("row_id should be Int32");+ assert_eq!(row_ids.values(), &[1, 2]);++ let (messages0, messages1) = extract_nested_list_values(kind, all_rows.column(1));++ let messages0 = messages0+ .as_any()+ .downcast_ref::<StructArray>()+ .expect("messages[0] should be a StructArray");+ let old_ids = messages0+ .column_by_name("id")+ .unwrap()+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap();+ assert_eq!(old_ids.values(), &[10, 20]);+ let old_chain = messages0+ .column_by_name("chain")+ .unwrap()+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap();+ assert_eq!(old_chain.iter().collect::<Vec<_>>(), vec![None, None]);++ let messages1 = messages1+ .as_any()+ .downcast_ref::<StructArray>()+ .expect("messages[1] should be a StructArray");+ assert!(+ messages1.column_by_name("ignored").is_none(),+ "extra source fields should not appear in the logical schema"+ );+ let new_chain = messages1+ .column_by_name("chain")+ .unwrap()+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap();+ assert_eq!(new_chain.iter().collect::<Vec<_>>(), vec![Some("eth")]);++ let projected = ctx+ .sql(+ "SELECT row_id, get_field(messages[1], 'id') AS msg_id, \+ get_field(messages[1], 'chain') AS chain \+ FROM t ORDER BY row_id",+ )+ .await?+ .collect()+ .await?;++ #[rustfmt::skip]+ let expected = [+ "+--------+--------+-------+",+ "| row_id | msg_id | chain |",+ "+--------+--------+-------+",+ "| 1 | 10 | |",+ "| 2 | 30 | eth |",+ "+--------+--------+-------+",+ ];+ assert_batches_eq!(expected, &projected);++ Ok(())+}+// Implement a custom PhysicalExprAdapterFactory that fills in missing columns with// the default value for the field type:// - Int64 columns are filled with `1`@@ -462,6 +863,125 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> {Ok(())}+/// Macro to generate paired test functions for List and LargeList variants.+/// Expands to two `#[tokio::test]` functions with the specified names.+macro_rules! test_struct_schema_evolution_pair {+ (+ list: $list_test:ident,+ large_list: $large_list_test:ident,+ fn: $assertion_fn:path $(, args: $($arg:expr),+)?+ ) => {+ #[tokio::test]+ async fn $list_test() {+ $assertion_fn(NestedListKind::List $(, $($arg),+)?).await;+ }++ #[tokio::test]+ async fn $large_list_test() {+ $assertion_fn(NestedListKind::LargeList $(, $($arg),+)?).await;+ }+ };+ (+ list: $list_test:ident,+ large_list: $large_list_test:ident,+ fn_result: $assertion_fn:path+ ) => {+ #[tokio::test]+ async fn $list_test() -> Result<()> {+ $assertion_fn(NestedListKind::List).await+ }++ #[tokio::test]+ async fn $large_list_test() -> Result<()> {+ $assertion_fn(NestedListKind::LargeList).await+ }+ };+}++test_struct_schema_evolution_pair!(+ list: test_list_struct_schema_evolution_end_to_end,+ large_list: test_large_list_struct_schema_evolution_end_to_end,+ fn_result: assert_nested_list_struct_schema_evolution+);++async fn assert_nested_list_struct_schema_evolution_errors(+ kind: NestedListKind,+ chain_type: DataType,+ chain_nullable: bool,+ expected_error: &str,+) {+ let batch = nested_messages_batch(+ kind,+ 1,+ &[NestedMessageRow {+ id: 10,+ name: "alpha",+ chain: Some("eth"),+ ignored: None,+ }],+ &message_fields(DataType::Utf8, true, true, false),+ );++ let table_schema =+ nested_list_table_schema(kind, target_message_fields(chain_type, chain_nullable));++ let ctx = setup_nested_list_test(+ kind,+ "struct_evolution_error",+ vec![("data.parquet".to_string(), batch)],+ table_schema,+ )+ .await;++ let err = ctx+ .sql("SELECT * FROM t")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap_err();+ assert!(+ err.to_string().contains(expected_error),+ "expected error to contain '{expected_error}', got: {err}"+ );+}++async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) {+ assert_nested_list_struct_schema_evolution_errors(+ kind,+ DataType::Utf8,+ false,+ "non-nullable",+ )+ .await;+}++async fn assert_incompatible_chain_field_fails(kind: NestedListKind) {+ assert_nested_list_struct_schema_evolution_errors(+ kind,+ incompatible_chain_type(),+ true,+ "Cannot cast struct field 'chain'",+ )+ .await;+}++fn incompatible_chain_type() -> DataType {+ DataType::Struct(vec![Arc::new(Field::new("value", DataType::Utf8, true))].into())+}++test_struct_schema_evolution_pair!(+ list: test_list_struct_schema_evolution_non_nullable_missing_field_fails,+ large_list: test_large_list_struct_schema_evolution_non_nullable_missing_field_fails,+ fn: assert_non_nullable_missing_chain_field_fails+);++test_struct_schema_evolution_pair!(+ list: test_list_struct_schema_evolution_incompatible_field_fails,+ large_list: test_large_list_struct_schema_evolution_incompatible_field_fails,+ fn: assert_incompatible_chain_field_fails+);+/// Test demonstrating that a single PhysicalExprAdapterFactory instance can be/// reused across multiple ListingTable instances.///diff --git a/datafusion/sqllogictest/test_files/schema_evolution_nested.slt b/datafusion/sqllogictest/test_files/schema_evolution_nested.sltnew file mode 100644index 0000000000..53bc16fe51--- /dev/null+++ b/datafusion/sqllogictest/test_files/schema_evolution_nested.slt@@ -0,0 +1,124 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++##########+# End-user-facing happy-path coverage for nested list/struct Parquet schema evolution.+#+# These tests generate mixed-schema parquet files through SQL COPY statements and+# query them through CREATE EXTERNAL TABLE, rather than constructing batches+# directly in Rust. Rust tests still cover more specialized fixture shapes and+# failure paths.+##########++statement ok+CREATE EXTERNAL TABLE list_messages (+ row_id INT,+ messages ARRAY<STRUCT<id BIGINT, name VARCHAR, chain VARCHAR>>+)+STORED AS PARQUET+LOCATION 'test_files/scratch/schema_evolution_nested/list_messages/';++statement ok+COPY (+ SELECT+ 1 AS row_id,+ [+ named_struct('id', 10, 'name', 'alpha'),+ named_struct('id', 20, 'name', 'beta')+ ] AS messages+) TO 'test_files/scratch/schema_evolution_nested/list_messages/old.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT+ 2 AS row_id,+ [+ named_struct('id', 30, 'name', 'gamma', 'chain', 'eth', 'ignored', 99)+ ] AS messages+) TO 'test_files/scratch/schema_evolution_nested/list_messages/new.parquet'+STORED AS PARQUET;++query I?+SELECT row_id, messages FROM list_messages ORDER BY row_id;+----+1 [{id: 10, name: alpha, chain: NULL}, {id: 20, name: beta, chain: NULL}]+2 [{id: 30, name: gamma, chain: eth}]++query IIT rowsort+SELECT+ row_id,+ get_field(messages[1], 'id') AS msg_id,+ get_field(messages[1], 'chain') AS chain+FROM list_messages;+----+1 10 NULL+2 30 eth++statement ok+COPY (+ SELECT+ 1 AS row_id,+ arrow_cast(+ [+ named_struct('id', 10, 'name', 'alpha'),+ named_struct('id', 20, 'name', 'beta')+ ],+ 'LargeList(Struct("id": Int64, "name": Utf8View))'+ ) AS messages+) TO 'test_files/scratch/schema_evolution_nested/large_list_messages/old.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT+ 2 AS row_id,+ arrow_cast(+ [+ named_struct('id', 30, 'name', 'gamma', 'chain', 'eth', 'ignored', 99)+ ],+ 'LargeList(Struct("id": Int64, "name": Utf8View, "chain": Utf8View, "ignored": Int64))'+ ) AS messages+) TO 'test_files/scratch/schema_evolution_nested/large_list_messages/new.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE large_list_messages+STORED AS PARQUET+LOCATION 'test_files/scratch/schema_evolution_nested/large_list_messages/';++query TTT+DESCRIBE large_list_messages;+----+row_id Int64 NO+messages LargeList(Struct("id": Int64, "name": Utf8View, "chain": Utf8View, "ignored": Int64)) NO++query I?+SELECT row_id, messages FROM large_list_messages ORDER BY row_id;+----+1 [{id: 10, name: alpha, chain: NULL, ignored: NULL}, {id: 20, name: beta, chain: NULL, ignored: NULL}]+2 [{id: 30, name: gamma, chain: eth, ignored: 99}]++query IIT rowsort+SELECT+ row_id,+ get_field(messages[1], 'id') AS msg_id,+ get_field(messages[1], 'chain') AS chain+FROM large_list_messages;+----+1 10 NULL+2 30 eth