Skip to content

Apache DataFusion SQL Query Engine

Add end-to-end Parquet tests for List and LargeList struct schema evolution (#20840)

kosiew6412c3aaa0e7
diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rs
index f412cdf9bd..cf32efbd70 100644
--- a/datafusion/core/tests/parquet/expr_adapter.rs
+++ b/datafusion/core/tests/parquet/expr_adapter.rs
@@ -18,9 +18,11 @@
use std::sync::Arc;
use arrow::array::{
- Array, ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatch, StringArray,
- StructArray, record_batch,
+ Array, ArrayRef, BooleanArray, Int32Array, Int64Array, LargeListArray, ListArray,
+ RecordBatch, StringArray, StructArray, record_batch,
};
+use arrow::buffer::OffsetBuffer;
+use arrow::compute::concat_batches;
use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef};
use bytes::{BufMut, BytesMut};
use datafusion::assert_batches_eq;
@@ -54,6 +56,405 @@ async fn write_parquet(batch: RecordBatch, store: Arc<dyn ObjectStore>, path: &s
store.put(&Path::from(path), data.into()).await.unwrap();
}
+#[derive(Debug, Clone, Copy)]
+enum NestedListKind {
+ List,
+ LargeList,
+}
+
+impl NestedListKind {
+ fn field_data_type(self, item_field: Arc<Field>) -> DataType {
+ match self {
+ Self::List => DataType::List(item_field),
+ Self::LargeList => DataType::LargeList(item_field),
+ }
+ }
+
+ fn array(
+ self,
+ item_field: Arc<Field>,
+ lengths: Vec<usize>,
+ values: ArrayRef,
+ ) -> ArrayRef {
+ match self {
+ Self::List => Arc::new(ListArray::new(
+ item_field,
+ OffsetBuffer::<i32>::from_lengths(lengths),
+ values,
+ None,
+ )),
+ Self::LargeList => Arc::new(LargeListArray::new(
+ item_field,
+ OffsetBuffer::<i64>::from_lengths(lengths),
+ values,
+ None,
+ )),
+ }
+ }
+
+ fn name(self) -> &'static str {
+ match self {
+ Self::List => "list",
+ Self::LargeList => "large_list",
+ }
+ }
+}
+
+#[derive(Debug)]
+// Fixture row for one nested struct element inside the `messages` list column.
+struct NestedMessageRow<'a> {
+ id: i32,
+ name: &'a str,
+ chain: Option<&'a str>,
+ ignored: Option<i32>,
+}
+
+fn message_fields(
+ chain_type: DataType,
+ chain_nullable: bool,
+ include_chain: bool,
+ include_ignored: bool,
+) -> Fields {
+ let mut fields = vec![
+ Arc::new(Field::new("id", DataType::Int32, false)),
+ Arc::new(Field::new("name", DataType::Utf8, true)),
+ ];
+ if include_chain {
+ fields.push(Arc::new(Field::new("chain", chain_type, chain_nullable)));
+ }
+ if include_ignored {
+ fields.push(Arc::new(Field::new("ignored", DataType::Int32, true)));
+ }
+ fields.into()
+}
+
+// Helper to construct the target message schema for struct evolution tests.
+// The schema always has id (Int64), name (Utf8), and chain with parameterized type.
+fn target_message_fields(chain_type: DataType, chain_nullable: bool) -> Fields {
+ vec![
+ Arc::new(Field::new("id", DataType::Int64, false)),
+ Arc::new(Field::new("name", DataType::Utf8, true)),
+ Arc::new(Field::new("chain", chain_type, chain_nullable)),
+ ]
+ .into()
+}
+
+// Helper to build message columns in canonical order (id, name, chain, ignored)
+// based on which optional fields are present in the schema.
+fn build_message_columns(
+ id_array: &ArrayRef,
+ name_array: &ArrayRef,
+ chain_vec: &[Option<&str>],
+ ignored_array: &ArrayRef,
+ fields: &Fields,
+) -> Vec<ArrayRef> {
+ let mut columns = vec![Arc::clone(id_array), Arc::clone(name_array)];
+
+ for field in fields.iter().skip(2) {
+ match field.name().as_str() {
+ "chain" => {
+ let chain_array = match field.data_type() {
+ DataType::Utf8 => {
+ Arc::new(StringArray::from(chain_vec.to_vec())) as ArrayRef
+ }
+ DataType::Struct(chain_fields) => {
+ let chain_struct = StructArray::new(
+ chain_fields.clone(),
+ vec![Arc::new(StringArray::from(chain_vec.to_vec()))
+ as ArrayRef],
+ None,
+ );
+ Arc::new(chain_struct) as ArrayRef
+ }
+ other => panic!("unexpected chain field type: {other:?}"),
+ };
+ columns.push(chain_array);
+ }
+ "ignored" => columns.push(Arc::clone(ignored_array)),
+ _ => {}
+ }
+ }
+ columns
+}
+
+fn nested_messages_batch(
+ kind: NestedListKind,
+ row_id: i32,
+ messages: &[NestedMessageRow<'_>],
+ fields: &Fields,
+) -> RecordBatch {
+ let item_field = Arc::new(Field::new("item", DataType::Struct(fields.clone()), true));
+
+ let (ids_vec, names_vec, chain_vec, ignored_vec) = messages.iter().fold(
+ (
+ Vec::with_capacity(messages.len()),
+ Vec::with_capacity(messages.len()),
+ Vec::with_capacity(messages.len()),
+ Vec::with_capacity(messages.len()),
+ ),
+ |(mut ids, mut names, mut chains, mut ignoreds), msg| {
+ ids.push(msg.id);
+ names.push(Some(msg.name));
+ chains.push(msg.chain);
+ ignoreds.push(msg.ignored);
+ (ids, names, chains, ignoreds)
+ },
+ );
+
+ // Build all arrays once
+ let id_array = Arc::new(Int32Array::from(ids_vec)) as ArrayRef;
+ let name_array = Arc::new(StringArray::from(names_vec)) as ArrayRef;
+ let ignored_array = Arc::new(Int32Array::from(ignored_vec)) as ArrayRef;
+
+ // Build columns in canonical order (id, name, chain, ignored) based on field schema
+ let columns =
+ build_message_columns(&id_array, &name_array, &chain_vec, &ignored_array, fields);
+
+ let struct_array = StructArray::new(fields.clone(), columns, None);
+
+ // Compute the message data type first, then move item_field into kind.array()
+ let message_data_type = kind.field_data_type(item_field.clone());
+ let messages_array =
+ kind.array(item_field, vec![messages.len()], Arc::new(struct_array));
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("row_id", DataType::Int32, false),
+ Field::new("messages", message_data_type, true),
+ ]));
+
+ RecordBatch::try_new(
+ schema,
+ vec![
+ Arc::new(Int32Array::from(vec![row_id])) as ArrayRef,
+ messages_array,
+ ],
+ )
+ .unwrap()
+}
+
+async fn register_memory_listing_table(
+ ctx: &SessionContext,
+ store: Arc<dyn ObjectStore>,
+ base_path: &str,
+ table_schema: SchemaRef,
+) {
+ let store_url = ObjectStoreUrl::parse("memory://").unwrap();
+ ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));
+
+ let listing_table_config =
+ ListingTableConfig::new(ListingTableUrl::parse(base_path).unwrap())
+ .infer_options(&ctx.state())
+ .await
+ .unwrap()
+ .with_schema(table_schema)
+ .with_expr_adapter_factory(Arc::new(DefaultPhysicalExprAdapterFactory));
+
+ let table = ListingTable::try_new(listing_table_config).unwrap();
+ ctx.register_table("t", Arc::new(table)).unwrap();
+}
+
+fn test_context() -> SessionContext {
+ let mut cfg = SessionConfig::new()
+ .with_collect_statistics(false)
+ .with_parquet_pruning(false)
+ .with_parquet_page_index_pruning(false);
+ cfg.options_mut().execution.parquet.pushdown_filters = true;
+ SessionContext::new_with_config(cfg)
+}
+
+fn nested_list_table_schema(
+ kind: NestedListKind,
+ target_message_fields: Fields,
+) -> SchemaRef {
+ let target_item = Arc::new(Field::new(
+ "item",
+ DataType::Struct(target_message_fields),
+ true,
+ ));
+ Arc::new(Schema::new(vec![
+ Field::new("row_id", DataType::Int32, false),
+ Field::new("messages", kind.field_data_type(target_item), true),
+ ]))
+}
+
+// Helper to extract message values from a nested list column.
+// Returns the values at indices 0 and 1 from either a ListArray or LargeListArray.
+fn extract_nested_list_values(
+ kind: NestedListKind,
+ column: &ArrayRef,
+) -> (ArrayRef, ArrayRef) {
+ match kind {
+ NestedListKind::List => {
+ let list = column
+ .as_any()
+ .downcast_ref::<ListArray>()
+ .expect("messages should be a ListArray");
+ (list.value(0), list.value(1))
+ }
+ NestedListKind::LargeList => {
+ let list = column
+ .as_any()
+ .downcast_ref::<LargeListArray>()
+ .expect("messages should be a LargeListArray");
+ (list.value(0), list.value(1))
+ }
+ }
+}
+
+// Helper to set up a nested list test fixture.
+// Creates an in-memory store, writes the provided batches to parquet files,
+// creates a SessionContext, and registers the resulting table.
+// Returns the prepared context ready for queries.
+async fn setup_nested_list_test(
+ kind: NestedListKind,
+ prefix_base: &str,
+ batches: Vec<(String, RecordBatch)>,
+ table_schema: SchemaRef,
+) -> SessionContext {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let prefix = format!("{}_{}", kind.name(), prefix_base);
+
+ for (filename, batch) in batches {
+ write_parquet(batch, Arc::clone(&store), &format!("{prefix}/{filename}")).await;
+ }
+
+ let ctx = test_context();
+ register_memory_listing_table(
+ &ctx,
+ Arc::clone(&store),
+ &format!("memory:///{prefix}/"),
+ table_schema,
+ )
+ .await;
+
+ ctx
+}
+
+async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Result<()> {
+ // old.parquet shape: messages item struct has only (id, name), no `chain`.
+ let old_batch = nested_messages_batch(
+ kind,
+ 1,
+ &[
+ NestedMessageRow {
+ id: 10,
+ name: "alpha",
+ chain: None,
+ ignored: None,
+ },
+ NestedMessageRow {
+ id: 20,
+ name: "beta",
+ chain: None,
+ ignored: None,
+ },
+ ],
+ &message_fields(DataType::Utf8, true, false, false),
+ );
+
+ // new.parquet shape: messages item struct adds nullable `chain` and extra `ignored`.
+ let new_batch = nested_messages_batch(
+ kind,
+ 2,
+ &[NestedMessageRow {
+ id: 30,
+ name: "gamma",
+ chain: Some("eth"),
+ ignored: Some(99),
+ }],
+ &message_fields(DataType::Utf8, true, true, true),
+ );
+
+ // Logical table schema expects evolved shape (id, name, nullable `chain`) and
+ // should ignore source-only `ignored` during reads.
+ let table_schema =
+ nested_list_table_schema(kind, target_message_fields(DataType::Utf8, true));
+
+ let ctx = setup_nested_list_test(
+ kind,
+ "struct_evolution",
+ vec![
+ ("old.parquet".to_string(), old_batch),
+ ("new.parquet".to_string(), new_batch),
+ ],
+ table_schema,
+ )
+ .await;
+
+ let select_all = ctx
+ .sql("SELECT * FROM t ORDER BY row_id")
+ .await?
+ .collect()
+ .await?;
+ let all_rows = concat_batches(&select_all[0].schema(), &select_all)?;
+
+ let row_ids = all_rows
+ .column(0)
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .expect("row_id should be Int32");
+ assert_eq!(row_ids.values(), &[1, 2]);
+
+ let (messages0, messages1) = extract_nested_list_values(kind, all_rows.column(1));
+
+ let messages0 = messages0
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .expect("messages[0] should be a StructArray");
+ let old_ids = messages0
+ .column_by_name("id")
+ .unwrap()
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .unwrap();
+ assert_eq!(old_ids.values(), &[10, 20]);
+ let old_chain = messages0
+ .column_by_name("chain")
+ .unwrap()
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .unwrap();
+ assert_eq!(old_chain.iter().collect::<Vec<_>>(), vec![None, None]);
+
+ let messages1 = messages1
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .expect("messages[1] should be a StructArray");
+ assert!(
+ messages1.column_by_name("ignored").is_none(),
+ "extra source fields should not appear in the logical schema"
+ );
+ let new_chain = messages1
+ .column_by_name("chain")
+ .unwrap()
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .unwrap();
+ assert_eq!(new_chain.iter().collect::<Vec<_>>(), vec![Some("eth")]);
+
+ let projected = ctx
+ .sql(
+ "SELECT row_id, get_field(messages[1], 'id') AS msg_id, \
+ get_field(messages[1], 'chain') AS chain \
+ FROM t ORDER BY row_id",
+ )
+ .await?
+ .collect()
+ .await?;
+
+ #[rustfmt::skip]
+ let expected = [
+ "+--------+--------+-------+",
+ "| row_id | msg_id | chain |",
+ "+--------+--------+-------+",
+ "| 1 | 10 | |",
+ "| 2 | 30 | eth |",
+ "+--------+--------+-------+",
+ ];
+ assert_batches_eq!(expected, &projected);
+
+ Ok(())
+}
+
// Implement a custom PhysicalExprAdapterFactory that fills in missing columns with
// the default value for the field type:
// - Int64 columns are filled with `1`
@@ -462,6 +863,125 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> {
Ok(())
}
+/// Macro to generate paired test functions for List and LargeList variants.
+/// Expands to two `#[tokio::test]` functions with the specified names.
+macro_rules! test_struct_schema_evolution_pair {
+ (
+ list: $list_test:ident,
+ large_list: $large_list_test:ident,
+ fn: $assertion_fn:path $(, args: $($arg:expr),+)?
+ ) => {
+ #[tokio::test]
+ async fn $list_test() {
+ $assertion_fn(NestedListKind::List $(, $($arg),+)?).await;
+ }
+
+ #[tokio::test]
+ async fn $large_list_test() {
+ $assertion_fn(NestedListKind::LargeList $(, $($arg),+)?).await;
+ }
+ };
+ (
+ list: $list_test:ident,
+ large_list: $large_list_test:ident,
+ fn_result: $assertion_fn:path
+ ) => {
+ #[tokio::test]
+ async fn $list_test() -> Result<()> {
+ $assertion_fn(NestedListKind::List).await
+ }
+
+ #[tokio::test]
+ async fn $large_list_test() -> Result<()> {
+ $assertion_fn(NestedListKind::LargeList).await
+ }
+ };
+}
+
+test_struct_schema_evolution_pair!(
+ list: test_list_struct_schema_evolution_end_to_end,
+ large_list: test_large_list_struct_schema_evolution_end_to_end,
+ fn_result: assert_nested_list_struct_schema_evolution
+);
+
+async fn assert_nested_list_struct_schema_evolution_errors(
+ kind: NestedListKind,
+ chain_type: DataType,
+ chain_nullable: bool,
+ expected_error: &str,
+) {
+ let batch = nested_messages_batch(
+ kind,
+ 1,
+ &[NestedMessageRow {
+ id: 10,
+ name: "alpha",
+ chain: Some("eth"),
+ ignored: None,
+ }],
+ &message_fields(DataType::Utf8, true, true, false),
+ );
+
+ let table_schema =
+ nested_list_table_schema(kind, target_message_fields(chain_type, chain_nullable));
+
+ let ctx = setup_nested_list_test(
+ kind,
+ "struct_evolution_error",
+ vec![("data.parquet".to_string(), batch)],
+ table_schema,
+ )
+ .await;
+
+ let err = ctx
+ .sql("SELECT * FROM t")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap_err();
+ assert!(
+ err.to_string().contains(expected_error),
+ "expected error to contain '{expected_error}', got: {err}"
+ );
+}
+
+async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) {
+ assert_nested_list_struct_schema_evolution_errors(
+ kind,
+ DataType::Utf8,
+ false,
+ "non-nullable",
+ )
+ .await;
+}
+
+async fn assert_incompatible_chain_field_fails(kind: NestedListKind) {
+ assert_nested_list_struct_schema_evolution_errors(
+ kind,
+ incompatible_chain_type(),
+ true,
+ "Cannot cast struct field 'chain'",
+ )
+ .await;
+}
+
+fn incompatible_chain_type() -> DataType {
+ DataType::Struct(vec![Arc::new(Field::new("value", DataType::Utf8, true))].into())
+}
+
+test_struct_schema_evolution_pair!(
+ list: test_list_struct_schema_evolution_non_nullable_missing_field_fails,
+ large_list: test_large_list_struct_schema_evolution_non_nullable_missing_field_fails,
+ fn: assert_non_nullable_missing_chain_field_fails
+);
+
+test_struct_schema_evolution_pair!(
+ list: test_list_struct_schema_evolution_incompatible_field_fails,
+ large_list: test_large_list_struct_schema_evolution_incompatible_field_fails,
+ fn: assert_incompatible_chain_field_fails
+);
+
/// Test demonstrating that a single PhysicalExprAdapterFactory instance can be
/// reused across multiple ListingTable instances.
///
diff --git a/datafusion/sqllogictest/test_files/schema_evolution_nested.slt b/datafusion/sqllogictest/test_files/schema_evolution_nested.slt
new file mode 100644
index 0000000000..53bc16fe51
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/schema_evolution_nested.slt
@@ -0,0 +1,124 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##########
+# End-user-facing happy-path coverage for nested list/struct Parquet schema evolution.
+#
+# These tests generate mixed-schema parquet files through SQL COPY statements and
+# query them through CREATE EXTERNAL TABLE, rather than constructing batches
+# directly in Rust. Rust tests still cover more specialized fixture shapes and
+# failure paths.
+##########
+
+statement ok
+CREATE EXTERNAL TABLE list_messages (
+ row_id INT,
+ messages ARRAY<STRUCT<id BIGINT, name VARCHAR, chain VARCHAR>>
+)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/schema_evolution_nested/list_messages/';
+
+statement ok
+COPY (
+ SELECT
+ 1 AS row_id,
+ [
+ named_struct('id', 10, 'name', 'alpha'),
+ named_struct('id', 20, 'name', 'beta')
+ ] AS messages
+) TO 'test_files/scratch/schema_evolution_nested/list_messages/old.parquet'
+STORED AS PARQUET;
+
+statement ok
+COPY (
+ SELECT
+ 2 AS row_id,
+ [
+ named_struct('id', 30, 'name', 'gamma', 'chain', 'eth', 'ignored', 99)
+ ] AS messages
+) TO 'test_files/scratch/schema_evolution_nested/list_messages/new.parquet'
+STORED AS PARQUET;
+
+query I?
+SELECT row_id, messages FROM list_messages ORDER BY row_id;
+----
+1 [{id: 10, name: alpha, chain: NULL}, {id: 20, name: beta, chain: NULL}]
+2 [{id: 30, name: gamma, chain: eth}]
+
+query IIT rowsort
+SELECT
+ row_id,
+ get_field(messages[1], 'id') AS msg_id,
+ get_field(messages[1], 'chain') AS chain
+FROM list_messages;
+----
+1 10 NULL
+2 30 eth
+
+statement ok
+COPY (
+ SELECT
+ 1 AS row_id,
+ arrow_cast(
+ [
+ named_struct('id', 10, 'name', 'alpha'),
+ named_struct('id', 20, 'name', 'beta')
+ ],
+ 'LargeList(Struct("id": Int64, "name": Utf8View))'
+ ) AS messages
+) TO 'test_files/scratch/schema_evolution_nested/large_list_messages/old.parquet'
+STORED AS PARQUET;
+
+statement ok
+COPY (
+ SELECT
+ 2 AS row_id,
+ arrow_cast(
+ [
+ named_struct('id', 30, 'name', 'gamma', 'chain', 'eth', 'ignored', 99)
+ ],
+ 'LargeList(Struct("id": Int64, "name": Utf8View, "chain": Utf8View, "ignored": Int64))'
+ ) AS messages
+) TO 'test_files/scratch/schema_evolution_nested/large_list_messages/new.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE large_list_messages
+STORED AS PARQUET
+LOCATION 'test_files/scratch/schema_evolution_nested/large_list_messages/';
+
+query TTT
+DESCRIBE large_list_messages;
+----
+row_id Int64 NO
+messages LargeList(Struct("id": Int64, "name": Utf8View, "chain": Utf8View, "ignored": Int64)) NO
+
+query I?
+SELECT row_id, messages FROM large_list_messages ORDER BY row_id;
+----
+1 [{id: 10, name: alpha, chain: NULL, ignored: NULL}, {id: 20, name: beta, chain: NULL, ignored: NULL}]
+2 [{id: 30, name: gamma, chain: eth, ignored: 99}]
+
+query IIT rowsort
+SELECT
+ row_id,
+ get_field(messages[1], 'id') AS msg_id,
+ get_field(messages[1], 'chain') AS chain
+FROM large_list_messages;
+----
+1 10 NULL
+2 30 eth