Apache DataFusion SQL Query Engine
Migrate Avro reader to arrow-avro and remove internal conversion code (#17861)
Namgung Chan627fabaa7795
diff --git a/Cargo.lock b/Cargo.lockindex be45541186..c7c0303c28 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -180,35 +180,6 @@ version = "1.0.102"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"-[[package]]-name = "apache-avro"-version = "0.21.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"-dependencies = [- "bigdecimal",- "bon",- "bzip2",- "crc32fast",- "digest",- "liblzma",- "log",- "miniz_oxide",- "num-bigint",- "quad-rand",- "rand 0.9.2",- "regex-lite",- "serde",- "serde_bytes",- "serde_json",- "snap",- "strum 0.27.2",- "strum_macros 0.27.2",- "thiserror",- "uuid",- "zstd",-]-[[package]]name = "ar_archive_writer"version = "0.5.1"@@ -286,6 +257,30 @@ dependencies = ["num-traits",]+[[package]]+name = "arrow-avro"+version = "58.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "36a3aadd016f63dfd4941ae8e13539ba98a3c2995adc3c88b9336d2514f6c8a7"+dependencies = [+ "arrow-array",+ "arrow-buffer",+ "arrow-schema",+ "bytes",+ "bzip2",+ "crc",+ "flate2",+ "indexmap 2.13.0",+ "liblzma",+ "rand 0.9.2",+ "serde",+ "serde_json",+ "snap",+ "strum_macros",+ "uuid",+ "zstd",+]+[[package]]name = "arrow-buffer"version = "58.1.0"@@ -1031,7 +1026,6 @@ dependencies = ["num-bigint","num-integer","num-traits",- "serde",][[package]]@@ -1146,31 +1140,6 @@ dependencies = ["time",]-[[package]]-name = "bon"-version = "3.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"-dependencies = [- "bon-macros",- "rustversion",-]--[[package]]-name = "bon-macros"-version = "3.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"-dependencies = [- "darling",- "ident_case",- "prettyplease",- "proc-macro2",- "quote",- "rustversion",- "syn 2.0.117",-]-[[package]]name = "brotli"version = "8.0.2"@@ -1524,6 +1493,21 @@ dependencies = ["libc",]+[[package]]+name = "crc"+version = "3.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"+dependencies = [+ "crc-catalog",+]++[[package]]+name = "crc-catalog"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"+[[package]]name = "crc32fast"version = "1.5.0"@@ -1891,7 +1875,6 @@ dependencies = [name = "datafusion-common"version = "53.0.0"dependencies = [- "apache-avro","arrow","arrow-ipc","chrono",@@ -1985,19 +1968,17 @@ dependencies = [name = "datafusion-datasource-avro"version = "53.0.0"dependencies = [- "apache-avro","arrow",+ "arrow-avro","async-trait","bytes","datafusion-common","datafusion-datasource",- "datafusion-physical-expr-common",+ "datafusion-physical-expr-adapter","datafusion-physical-plan","datafusion-session","futures",- "num-traits","object_store",- "serde_json",][[package]]@@ -2109,8 +2090,8 @@ dependencies = ["rand 0.9.2","serde","serde_json",- "strum 0.28.0",- "strum_macros 0.28.0",+ "strum",+ "strum_macros","tempfile","test-utils","tokio",@@ -4167,7 +4148,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"dependencies = ["num-integer","num-traits",- "serde",][[package]]@@ -4828,12 +4808,6 @@ dependencies = ["cc",]-[[package]]-name = "quad-rand"-version = "0.2.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"-[[package]]name = "quick-xml"version = "0.38.4"@@ -5477,16 +5451,6 @@ dependencies = ["serde_derive",]-[[package]]-name = "serde_bytes"-version = "0.11.19"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"-dependencies = [- "serde",- "serde_core",-]-[[package]]name = "serde_core"version = "1.0.228"@@ -5834,30 +5798,12 @@ dependencies = ["syn 2.0.117",]-[[package]]-name = "strum"-version = "0.27.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"-[[package]]name = "strum"version = "0.28.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"-[[package]]-name = "strum_macros"-version = "0.27.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"-dependencies = [- "heck",- "proc-macro2",- "quote",- "syn 2.0.117",-]-[[package]]name = "strum_macros"version = "0.28.0"@@ -6646,7 +6592,6 @@ checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"dependencies = ["getrandom 0.4.2","js-sys",- "serde_core","wasm-bindgen",]diff --git a/Cargo.toml b/Cargo.tomlindex 6c88a7fef1..e0fdf93e85 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -92,6 +92,13 @@ arrow = { version = "58.1.0", features = ["prettyprint","chrono-tz",] }+arrow-avro = { version = "58.1.0", default-features = false, features = [+ "deflate",+ "snappy",+ "zstd",+ "bzip2",+ "xz",+] }arrow-buffer = { version = "58.1.0", default-features = false }arrow-flight = { version = "58.1.0", features = ["flight-sql-experimental",diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.tomlindex 92dd76aa97..c035744419 100644--- a/datafusion/common/Cargo.toml+++ b/datafusion/common/Cargo.toml@@ -41,7 +41,6 @@ workspace = truename = "datafusion_common"[features]-avro = ["apache-avro"]backtrace = []parquet_encryption = ["parquet",@@ -66,12 +65,6 @@ harness = falsename = "stats_merge"[dependencies]-apache-avro = { workspace = true, features = [- "bzip",- "snappy",- "xz",- "zstandard",-], optional = true }arrow = { workspace = true }arrow-ipc = { workspace = true }chrono = { workspace = true }diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rsindex b7a30f868a..fbc1421727 100644--- a/datafusion/common/src/error.rs+++ b/datafusion/common/src/error.rs@@ -48,8 +48,6 @@ use std::sync::Arc;use crate::utils::datafusion_strsim::normalized_levenshtein;use crate::utils::quote_identifier;use crate::{Column, DFSchema, Diagnostic, TableReference};-#[cfg(feature = "avro")]-use apache_avro::Error as AvroError;use arrow::error::ArrowError;#[cfg(feature = "parquet")]use parquet::errors::ParquetError;@@ -76,9 +74,6 @@ pub enum DataFusionError {/// Error when reading / writing Parquet data.#[cfg(feature = "parquet")]ParquetError(Box<ParquetError>),- /// Error when reading Avro data.- #[cfg(feature = "avro")]- AvroError(Box<AvroError>),/// Error when reading / writing to / from an object_store (e.g. S3 or LocalFile)#[cfg(feature = "object_store")]ObjectStore(Box<object_store::Error>),@@ -332,13 +327,6 @@ impl From<ParquetError> for DataFusionError {}}-#[cfg(feature = "avro")]-impl From<AvroError> for DataFusionError {- fn from(e: AvroError) -> Self {- DataFusionError::AvroError(Box::new(e))- }-}-#[cfg(feature = "object_store")]impl From<object_store::Error> for DataFusionError {fn from(e: object_store::Error) -> Self {@@ -389,8 +377,6 @@ impl Error for DataFusionError {DataFusionError::ArrowError(e, _) => Some(e.as_ref()),#[cfg(feature = "parquet")]DataFusionError::ParquetError(e) => Some(e.as_ref()),- #[cfg(feature = "avro")]- DataFusionError::AvroError(e) => Some(e.as_ref()),#[cfg(feature = "object_store")]DataFusionError::ObjectStore(e) => Some(e.as_ref()),DataFusionError::IoError(e) => Some(e),@@ -520,8 +506,6 @@ impl DataFusionError {DataFusionError::ArrowError(_, _) => "Arrow error: ",#[cfg(feature = "parquet")]DataFusionError::ParquetError(_) => "Parquet error: ",- #[cfg(feature = "avro")]- DataFusionError::AvroError(_) => "Avro error: ",#[cfg(feature = "object_store")]DataFusionError::ObjectStore(_) => "Object Store error: ",DataFusionError::IoError(_) => "IO error: ",@@ -561,8 +545,6 @@ impl DataFusionError {}#[cfg(feature = "parquet")]DataFusionError::ParquetError(ref desc) => Cow::Owned(desc.to_string()),- #[cfg(feature = "avro")]- DataFusionError::AvroError(ref desc) => Cow::Owned(desc.to_string()),DataFusionError::IoError(ref desc) => Cow::Owned(desc.to_string()),#[cfg(feature = "sql")]DataFusionError::SQL(ref desc, ref backtrace) => {diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex e3864e4bff..c1230f7d5d 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -43,7 +43,7 @@ nested_expressions = ["datafusion-functions-nested"]# This feature is deprecated. Use the `nested_expressions` feature instead.array_expressions = ["nested_expressions"]# Used to enable the avro format-avro = ["datafusion-common/avro", "datafusion-datasource-avro"]+avro = ["datafusion-datasource-avro"]backtrace = ["datafusion-common/backtrace"]compression = ["liblzma",diff --git a/datafusion/core/src/datasource/file_format/avro.rs b/datafusion/core/src/datasource/file_format/avro.rsindex 7cf23ee294..a8b48cc736 100644--- a/datafusion/core/src/datasource/file_format/avro.rs+++ b/datafusion/core/src/datasource/file_format/avro.rs@@ -109,7 +109,7 @@ mod tests {"double_col: Float64","date_string_col: Binary","string_col: Binary",- "timestamp_col: Timestamp(µs)",+ "timestamp_col: Timestamp(µs, \"+00:00\")",],x);@@ -118,18 +118,18 @@ mod tests {assert_eq!(batches.len(), 1);assert_snapshot!(batches_to_string(&batches),@r"- +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+- | id | bool_col | tinyint_col | smallint_col | int_col | bigint_col | float_col | double_col | date_string_col | string_col | timestamp_col |- +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+- | 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00 |- | 5 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30332f30312f3039 | 31 | 2009-03-01T00:01:00 |- | 6 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30342f30312f3039 | 30 | 2009-04-01T00:00:00 |- | 7 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30342f30312f3039 | 31 | 2009-04-01T00:01:00 |- | 2 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30322f30312f3039 | 30 | 2009-02-01T00:00:00 |- | 3 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30322f30312f3039 | 31 | 2009-02-01T00:01:00 |- | 0 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30312f30312f3039 | 30 | 2009-01-01T00:00:00 |- | 1 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30312f30312f3039 | 31 | 2009-01-01T00:01:00 |- +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------++ +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+----------------------++ | id | bool_col | tinyint_col | smallint_col | int_col | bigint_col | float_col | double_col | date_string_col | string_col | timestamp_col |+ +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+----------------------++ | 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00Z |+ | 5 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30332f30312f3039 | 31 | 2009-03-01T00:01:00Z |+ | 6 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30342f30312f3039 | 30 | 2009-04-01T00:00:00Z |+ | 7 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30342f30312f3039 | 31 | 2009-04-01T00:01:00Z |+ | 2 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30322f30312f3039 | 30 | 2009-02-01T00:00:00Z |+ | 3 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30322f30312f3039 | 31 | 2009-02-01T00:01:00Z |+ | 0 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30312f30312f3039 | 30 | 2009-01-01T00:00:00Z |+ | 1 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30312f30312f3039 | 31 | 2009-01-01T00:01:00Z |+ +----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+----------------------+");Ok(())}diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rsindex 349eee5592..e7e6be4f2b 100644--- a/datafusion/core/src/lib.rs+++ b/datafusion/core/src/lib.rs@@ -786,7 +786,7 @@ pub use object_store;pub use parquet;#[cfg(feature = "avro")]-pub use datafusion_datasource_avro::apache_avro;+pub use datafusion_datasource_avro::arrow_avro;// re-export DataFusion sub-crates at the top level. Use `pub use *`// so that the contents of the subcrates appears in rustdocsdiff --git a/datafusion/datasource-avro/Cargo.toml b/datafusion/datasource-avro/Cargo.tomlindex c9299aeb10..adc2be1cb8 100644--- a/datafusion/datasource-avro/Cargo.toml+++ b/datafusion/datasource-avro/Cargo.toml@@ -31,21 +31,19 @@ version.workspace = trueall-features = true[dependencies]-apache-avro = { workspace = true }arrow = { workspace = true }+arrow-avro = { workspace = true }async-trait = { workspace = true }bytes = { workspace = true }-datafusion-common = { workspace = true, features = ["object_store", "avro"] }+datafusion-common = { workspace = true, features = ["object_store"] }datafusion-datasource = { workspace = true }-datafusion-physical-expr-common = { workspace = true }+datafusion-physical-expr-adapter = { workspace = true }datafusion-physical-plan = { workspace = true }datafusion-session = { workspace = true }futures = { workspace = true }-num-traits = { workspace = true }object_store = { workspace = true }[dev-dependencies]-serde_json = { workspace = true }# Note: add additional linter rules in lib.rs.# Rust does not support workspace + new linter rules in subcrates yetdiff --git a/datafusion/datasource-avro/src/avro_to_arrow/arrow_array_reader.rs b/datafusion/datasource-avro/src/avro_to_arrow/arrow_array_reader.rsdeleted file mode 100644index ea676a7611..0000000000--- a/datafusion/datasource-avro/src/avro_to_arrow/arrow_array_reader.rs+++ /dev/null@@ -1,1807 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--//! Avro to Arrow array readers--use apache_avro::schema::RecordSchema;-use apache_avro::{- Error as AvroError, Reader as AvroReader,- error::Details as AvroErrorDetails,- schema::{Schema as AvroSchema, SchemaKind},- types::Value,-};-use arrow::array::{- Array, ArrayBuilder, ArrayData, ArrayDataBuilder, ArrayRef, BooleanBuilder,- LargeStringArray, ListBuilder, NullArray, OffsetSizeTrait, PrimitiveArray,- StringArray, StringBuilder, StringDictionaryBuilder, make_array,-};-use arrow::array::{BinaryArray, FixedSizeBinaryArray, GenericListArray};-use arrow::buffer::{Buffer, MutableBuffer};-use arrow::datatypes::{- ArrowDictionaryKeyType, ArrowNumericType, ArrowPrimitiveType, DataType, Date32Type,- Date64Type, Field, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type,- Int64Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,- Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,- TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type,- UInt64Type,-};-use arrow::datatypes::{Fields, SchemaRef};-use arrow::error::ArrowError;-use arrow::error::ArrowError::SchemaError;-use arrow::error::Result as ArrowResult;-use arrow::record_batch::RecordBatch;-use arrow::util::bit_util;-use datafusion_common::arrow_err;-use datafusion_common::error::Result;-use num_traits::NumCast;-use std::collections::BTreeMap;-use std::io::Read;-use std::sync::Arc;--type RecordSlice<'a> = &'a [&'a Vec<(String, Value)>];--pub struct AvroArrowArrayReader<'a, R: Read> {- reader: AvroReader<'a, R>,- schema: SchemaRef,- schema_lookup: BTreeMap<String, usize>,-}--impl<R: Read> AvroArrowArrayReader<'_, R> {- pub fn try_new(reader: R, schema: SchemaRef) -> Result<Self> {- let reader = AvroReader::new(reader)?;- let writer_schema = reader.writer_schema().clone();- let schema_lookup = Self::schema_lookup(writer_schema)?;- Ok(Self {- reader,- schema,- schema_lookup,- })- }-- pub fn schema_lookup(schema: AvroSchema) -> Result<BTreeMap<String, usize>> {- match schema {- AvroSchema::Record(RecordSchema {- fields, mut lookup, ..- }) => {- for field in fields {- Self::child_schema_lookup(&field.name, &field.schema, &mut lookup)?;- }- Ok(lookup)- }- _ => arrow_err!(SchemaError(- "expected avro schema to be a record".to_string(),- )),- }- }-- fn child_schema_lookup<'b>(- parent_field_name: &str,- schema: &AvroSchema,- schema_lookup: &'b mut BTreeMap<String, usize>,- ) -> Result<&'b BTreeMap<String, usize>> {- match schema {- AvroSchema::Union(us) => {- let has_nullable = us- .find_schema_with_known_schemata::<apache_avro::Schema>(- &Value::Null,- None,- &None,- )- .is_some();- let sub_schemas = us.variants();- if has_nullable- && sub_schemas.len() == 2- && let Some(sub_schema) =- sub_schemas.iter().find(|&s| !matches!(s, AvroSchema::Null))- {- Self::child_schema_lookup(- parent_field_name,- sub_schema,- schema_lookup,- )?;- }- }- AvroSchema::Record(RecordSchema { fields, lookup, .. }) => {- lookup.iter().for_each(|(field_name, pos)| {- schema_lookup- .insert(format!("{parent_field_name}.{field_name}"), *pos);- });-- for field in fields {- let sub_parent_field_name =- format!("{}.{}", parent_field_name, field.name);- Self::child_schema_lookup(- &sub_parent_field_name,- &field.schema,- schema_lookup,- )?;- }- }- AvroSchema::Array(schema) => {- Self::child_schema_lookup(- parent_field_name,- &schema.items,- schema_lookup,- )?;- }- _ => (),- }- Ok(schema_lookup)- }-- /// Read the next batch of records- pub fn next_batch(&mut self, batch_size: usize) -> Option<ArrowResult<RecordBatch>> {- let rows_result = self- .reader- .by_ref()- .take(batch_size)- .map(|value| match value {- Ok(Value::Record(v)) => Ok(v),- Err(e) => Err(ArrowError::ParseError(format!(- "Failed to parse avro value: {e}"- ))),- other => Err(ArrowError::ParseError(format!(- "Row needs to be of type object, got: {other:?}"- ))),- })- .collect::<ArrowResult<Vec<Vec<(String, Value)>>>>();-- let rows = match rows_result {- // Return error early- Err(e) => return Some(Err(e)),- // No rows: return None early- Ok(rows) if rows.is_empty() => return None,- Ok(rows) => rows,- };-- let rows = rows.iter().collect::<Vec<&Vec<(String, Value)>>>();- let arrays = self.build_struct_array(&rows, "", self.schema.fields());-- Some(arrays.and_then(|arr| RecordBatch::try_new(Arc::clone(&self.schema), arr)))- }-- fn build_boolean_array(&self, rows: RecordSlice, col_name: &str) -> ArrayRef {- let mut builder = BooleanBuilder::with_capacity(rows.len());- for row in rows {- if let Some(value) = self.field_lookup(col_name, row) {- if let Some(boolean) = resolve_boolean(value) {- builder.append_value(boolean)- } else {- builder.append_null();- }- } else {- builder.append_null();- }- }- Arc::new(builder.finish())- }-- fn build_primitive_array<T>(&self, rows: RecordSlice, col_name: &str) -> ArrayRef- where- T: ArrowNumericType + Resolver,- T::Native: NumCast,- {- Arc::new(- rows.iter()- .map(|row| {- self.field_lookup(col_name, row)- .and_then(|value| resolve_item::<T>(value))- })- .collect::<PrimitiveArray<T>>(),- )- }-- #[inline(always)]- fn build_string_dictionary_builder<T>(- &self,- row_len: usize,- ) -> StringDictionaryBuilder<T>- where- T: ArrowPrimitiveType + ArrowDictionaryKeyType,- {- StringDictionaryBuilder::with_capacity(row_len, row_len, row_len)- }-- fn build_wrapped_list_array(- &self,- rows: RecordSlice,- col_name: &str,- key_type: &DataType,- ) -> ArrowResult<ArrayRef> {- match *key_type {- DataType::Int8 => {- let dtype = DataType::Dictionary(- Box::new(DataType::Int8),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<Int8Type>(&dtype, col_name, rows)- }- DataType::Int16 => {- let dtype = DataType::Dictionary(- Box::new(DataType::Int16),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<Int16Type>(&dtype, col_name, rows)- }- DataType::Int32 => {- let dtype = DataType::Dictionary(- Box::new(DataType::Int32),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<Int32Type>(&dtype, col_name, rows)- }- DataType::Int64 => {- let dtype = DataType::Dictionary(- Box::new(DataType::Int64),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<Int64Type>(&dtype, col_name, rows)- }- DataType::UInt8 => {- let dtype = DataType::Dictionary(- Box::new(DataType::UInt8),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<UInt8Type>(&dtype, col_name, rows)- }- DataType::UInt16 => {- let dtype = DataType::Dictionary(- Box::new(DataType::UInt16),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<UInt16Type>(&dtype, col_name, rows)- }- DataType::UInt32 => {- let dtype = DataType::Dictionary(- Box::new(DataType::UInt32),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<UInt32Type>(&dtype, col_name, rows)- }- DataType::UInt64 => {- let dtype = DataType::Dictionary(- Box::new(DataType::UInt64),- Box::new(DataType::Utf8),- );- self.list_array_string_array_builder::<UInt64Type>(&dtype, col_name, rows)- }- ref e => Err(SchemaError(format!(- "Data type is currently not supported for dictionaries in list : {e}"- ))),- }- }-- #[inline(always)]- fn list_array_string_array_builder<D>(- &self,- data_type: &DataType,- col_name: &str,- rows: RecordSlice,- ) -> ArrowResult<ArrayRef>- where- D: ArrowPrimitiveType + ArrowDictionaryKeyType,- {- let mut builder: Box<dyn ArrayBuilder> = match data_type {- DataType::Utf8 => {- let values_builder = StringBuilder::with_capacity(rows.len(), 5);- Box::new(ListBuilder::new(values_builder))- }- DataType::Dictionary(_, _) => {- let values_builder =- self.build_string_dictionary_builder::<D>(rows.len() * 5);- Box::new(ListBuilder::new(values_builder))- }- e => {- return Err(SchemaError(format!(- "Nested list data builder type is not supported: {e}"- )));- }- };-- for row in rows {- if let Some(value) = self.field_lookup(col_name, row) {- let value = maybe_resolve_union(value);- // value can be an array or a scalar- let vals: Vec<Option<String>> = if let Value::String(v) = value {- vec![Some(v.to_string())]- } else if let Value::Array(n) = value {- n.iter()- .map(resolve_string)- .collect::<ArrowResult<Vec<Option<String>>>>()?- .into_iter()- .collect::<Vec<Option<String>>>()- } else if let Value::Null = value {- vec![None]- } else if !matches!(value, Value::Record(_)) {- vec![resolve_string(value)?]- } else {- return Err(SchemaError(- "Only scalars are currently supported in Avro arrays".to_string(),- ));- };-- // TODO: ARROW-10335: APIs of dictionary arrays and others are different. Unify- // them.- match data_type {- DataType::Utf8 => {- let builder = builder- .as_any_mut()- .downcast_mut::<ListBuilder<StringBuilder>>()- .ok_or_else(||SchemaError(- "Cast failed for ListBuilder<StringBuilder> during nested data parsing".to_string(),- ))?;- for val in vals {- if let Some(v) = val {- builder.values().append_value(&v)- } else {- builder.values().append_null()- };- }-- // Append to the list- builder.append(true);- }- DataType::Dictionary(_, _) => {- let builder = builder.as_any_mut().downcast_mut::<ListBuilder<StringDictionaryBuilder<D>>>().ok_or_else(||SchemaError(- "Cast failed for ListBuilder<StringDictionaryBuilder> during nested data parsing".to_string(),- ))?;- for val in vals {- if let Some(v) = val {- let _ = builder.values().append(&v)?;- } else {- builder.values().append_null()- };- }-- // Append to the list- builder.append(true);- }- e => {- return Err(SchemaError(format!(- "Nested list data builder type is not supported: {e}"- )));- }- }- }- }-- Ok(builder.finish() as ArrayRef)- }-- #[inline(always)]- fn build_dictionary_array<T>(- &self,- rows: RecordSlice,- col_name: &str,- ) -> ArrowResult<ArrayRef>- where- T::Native: NumCast,- T: ArrowPrimitiveType + ArrowDictionaryKeyType,- {- let mut builder: StringDictionaryBuilder<T> =- self.build_string_dictionary_builder(rows.len());- for row in rows {- if let Some(value) = self.field_lookup(col_name, row) {- if let Ok(Some(str_v)) = resolve_string(value) {- builder.append(str_v).map(drop)?- } else {- builder.append_null()- }- } else {- builder.append_null()- }- }- Ok(Arc::new(builder.finish()) as ArrayRef)- }-- #[inline(always)]- fn build_string_dictionary_array(- &self,- rows: RecordSlice,- col_name: &str,- key_type: &DataType,- value_type: &DataType,- ) -> ArrowResult<ArrayRef> {- if let DataType::Utf8 = *value_type {- match *key_type {- DataType::Int8 => self.build_dictionary_array::<Int8Type>(rows, col_name),- DataType::Int16 => {- self.build_dictionary_array::<Int16Type>(rows, col_name)- }- DataType::Int32 => {- self.build_dictionary_array::<Int32Type>(rows, col_name)- }- DataType::Int64 => {- self.build_dictionary_array::<Int64Type>(rows, col_name)- }- DataType::UInt8 => {- self.build_dictionary_array::<UInt8Type>(rows, col_name)- }- DataType::UInt16 => {- self.build_dictionary_array::<UInt16Type>(rows, col_name)- }- DataType::UInt32 => {- self.build_dictionary_array::<UInt32Type>(rows, col_name)- }- DataType::UInt64 => {- self.build_dictionary_array::<UInt64Type>(rows, col_name)- }- _ => Err(SchemaError("unsupported dictionary key type".to_string())),- }- } else {- Err(SchemaError(- "dictionary types other than UTF-8 not yet supported".to_string(),- ))- }- }-- /// Build a nested GenericListArray from a list of unnested `Value`s- fn build_nested_list_array<OffsetSize: OffsetSizeTrait>(- &self,- parent_field_name: &str,- rows: &[&Value],- list_field: &Field,- ) -> ArrowResult<ArrayRef> {- // build list offsets- let mut cur_offset = OffsetSize::zero();- let list_len = rows.len();- let num_list_bytes = bit_util::ceil(list_len, 8);- let mut offsets = Vec::with_capacity(list_len + 1);- let mut list_nulls = MutableBuffer::from_len_zeroed(num_list_bytes);- offsets.push(cur_offset);- rows.iter().enumerate().for_each(|(i, v)| {- // TODO: unboxing Union(Array(Union(...))) should probably be done earlier- let v = maybe_resolve_union(v);- if let Value::Array(a) = v {- cur_offset += OffsetSize::from_usize(a.len()).unwrap();- bit_util::set_bit(&mut list_nulls, i);- } else if let Value::Null = v {- // value is null, not incremented- } else {- cur_offset += OffsetSize::one();- }- offsets.push(cur_offset);- });- let valid_len = cur_offset.to_usize().unwrap();- let array_data = match list_field.data_type() {- DataType::Null => NullArray::new(valid_len).into_data(),- DataType::Boolean => {- let num_bytes = bit_util::ceil(valid_len, 8);- let mut bool_values = MutableBuffer::from_len_zeroed(num_bytes);- let mut bool_nulls =- MutableBuffer::new(num_bytes).with_bitset(num_bytes, true);- let mut curr_index = 0;- rows.iter().for_each(|v| {- if let Value::Array(vs) = v {- vs.iter().for_each(|value| {- if let Value::Boolean(child) = value {- // if valid boolean, append value- if *child {- bit_util::set_bit(&mut bool_values, curr_index);- }- } else {- // null slot- bit_util::unset_bit(&mut bool_nulls, curr_index);- }- curr_index += 1;- });- }- });- ArrayData::builder(list_field.data_type().clone())- .len(valid_len)- .add_buffer(bool_values.into())- .null_bit_buffer(Some(bool_nulls.into()))- .build()- .unwrap()- }- DataType::Int8 => self.read_primitive_list_values::<Int8Type>(rows),- DataType::Int16 => self.read_primitive_list_values::<Int16Type>(rows),- DataType::Int32 => self.read_primitive_list_values::<Int32Type>(rows),- DataType::Int64 => self.read_primitive_list_values::<Int64Type>(rows),- DataType::UInt8 => self.read_primitive_list_values::<UInt8Type>(rows),- DataType::UInt16 => self.read_primitive_list_values::<UInt16Type>(rows),- DataType::UInt32 => self.read_primitive_list_values::<UInt32Type>(rows),- DataType::UInt64 => self.read_primitive_list_values::<UInt64Type>(rows),- DataType::Float16 => {- return Err(SchemaError("Float16 not supported".to_string()));- }- DataType::Float32 => self.read_primitive_list_values::<Float32Type>(rows),- DataType::Float64 => self.read_primitive_list_values::<Float64Type>(rows),- DataType::Timestamp(_, _)- | DataType::Date32- | DataType::Date64- | DataType::Time32(_)- | DataType::Time64(_) => {- return Err(SchemaError(- "Temporal types are not yet supported, see ARROW-4803".to_string(),- ));- }- DataType::Utf8 => flatten_string_values(rows)- .into_iter()- .collect::<StringArray>()- .into_data(),- DataType::LargeUtf8 => flatten_string_values(rows)- .into_iter()- .collect::<LargeStringArray>()- .into_data(),- DataType::List(field) => {- let child = self.build_nested_list_array::<i32>(- parent_field_name,- &flatten_values(rows),- field,- )?;- child.to_data()- }- DataType::LargeList(field) => {- let child = self.build_nested_list_array::<i64>(- parent_field_name,- &flatten_values(rows),- field,- )?;- child.to_data()- }- DataType::Struct(fields) => {- // extract list values, with non-lists converted to Value::Null- let array_item_count = rows- .iter()- .map(|row| match maybe_resolve_union(row) {- Value::Array(values) => values.len(),- _ => 1,- })- .sum();- let num_bytes = bit_util::ceil(array_item_count, 8);- let mut null_buffer = MutableBuffer::from_len_zeroed(num_bytes);- let mut struct_index = 0;- let null_struct_array = vec![("null".to_string(), Value::Null)];- let rows: Vec<&Vec<(String, Value)>> = rows- .iter()- .map(|v| maybe_resolve_union(v))- .flat_map(|row| {- if let Value::Array(values) = row {- values- .iter()- .map(maybe_resolve_union)- .map(|v| match v {- Value::Record(record) => {- bit_util::set_bit(&mut null_buffer, struct_index);- struct_index += 1;- record- }- Value::Null => {- struct_index += 1;- &null_struct_array- }- other => panic!("expected Record, got {other:?}"),- })- .collect::<Vec<&Vec<(String, Value)>>>()- } else {- struct_index += 1;- vec![&null_struct_array]- }- })- .collect();-- let arrays = self.build_struct_array(&rows, parent_field_name, fields)?;- let data_type = DataType::Struct(fields.clone());- ArrayDataBuilder::new(data_type)- .len(rows.len())- .null_bit_buffer(Some(null_buffer.into()))- .child_data(arrays.into_iter().map(|a| a.to_data()).collect())- .build()- .unwrap()- }- datatype => {- return Err(SchemaError(format!(- "Nested list of {datatype} not supported"- )));- }- };- // build list- let list_data = ArrayData::builder(DataType::List(Arc::new(list_field.clone())))- .len(list_len)- .add_buffer(Buffer::from_slice_ref(&offsets))- .add_child_data(array_data)- .null_bit_buffer(Some(list_nulls.into()))- .build()- .unwrap();- Ok(Arc::new(GenericListArray::<OffsetSize>::from(list_data)))- }-- /// Builds the child values of a `StructArray`, falling short of constructing the StructArray.- /// The function does not construct the StructArray as some callers would want the child arrays.- ///- /// *Note*: The function is recursive, and will read nested structs.- fn build_struct_array(- &self,- rows: RecordSlice,- parent_field_name: &str,- struct_fields: &Fields,- ) -> ArrowResult<Vec<ArrayRef>> {- let arrays: ArrowResult<Vec<ArrayRef>> = struct_fields- .iter()- .map(|field| {- let field_path = if parent_field_name.is_empty() {- field.name().to_string()- } else {- format!("{}.{}", parent_field_name, field.name())- };- let arr = match field.data_type() {- DataType::Null => Arc::new(NullArray::new(rows.len())) as ArrayRef,- DataType::Boolean => self.build_boolean_array(rows, &field_path),- DataType::Float64 => {- self.build_primitive_array::<Float64Type>(rows, &field_path)- }- DataType::Float32 => {- self.build_primitive_array::<Float32Type>(rows, &field_path)- }- DataType::Int64 => {- self.build_primitive_array::<Int64Type>(rows, &field_path)- }- DataType::Int32 => {- self.build_primitive_array::<Int32Type>(rows, &field_path)- }- DataType::Int16 => {- self.build_primitive_array::<Int16Type>(rows, &field_path)- }- DataType::Int8 => {- self.build_primitive_array::<Int8Type>(rows, &field_path)- }- DataType::UInt64 => {- self.build_primitive_array::<UInt64Type>(rows, &field_path)- }- DataType::UInt32 => {- self.build_primitive_array::<UInt32Type>(rows, &field_path)- }- DataType::UInt16 => {- self.build_primitive_array::<UInt16Type>(rows, &field_path)- }- DataType::UInt8 => {- self.build_primitive_array::<UInt8Type>(rows, &field_path)- }- // TODO: this is incomplete- DataType::Timestamp(unit, _) => match unit {- TimeUnit::Second => self- .build_primitive_array::<TimestampSecondType>(- rows,- &field_path,- ),- TimeUnit::Microsecond => self- .build_primitive_array::<TimestampMicrosecondType>(- rows,- &field_path,- ),- TimeUnit::Millisecond => self- .build_primitive_array::<TimestampMillisecondType>(- rows,- &field_path,- ),- TimeUnit::Nanosecond => self- .build_primitive_array::<TimestampNanosecondType>(- rows,- &field_path,- ),- },- DataType::Date64 => {- self.build_primitive_array::<Date64Type>(rows, &field_path)- }- DataType::Date32 => {- self.build_primitive_array::<Date32Type>(rows, &field_path)- }- DataType::Time64(unit) => match unit {- TimeUnit::Microsecond => self- .build_primitive_array::<Time64MicrosecondType>(- rows,- &field_path,- ),- TimeUnit::Nanosecond => self- .build_primitive_array::<Time64NanosecondType>(- rows,- &field_path,- ),- t => {- return Err(SchemaError(format!(- "TimeUnit {t:?} not supported with Time64"- )));- }- },- DataType::Time32(unit) => match unit {- TimeUnit::Second => self- .build_primitive_array::<Time32SecondType>(rows, &field_path),- TimeUnit::Millisecond => self- .build_primitive_array::<Time32MillisecondType>(- rows,- &field_path,- ),- t => {- return Err(SchemaError(format!(- "TimeUnit {t:?} not supported with Time32"- )));- }- },- DataType::Utf8 | DataType::LargeUtf8 => Arc::new(- rows.iter()- .map(|row| {- let maybe_value = self.field_lookup(&field_path, row);- match maybe_value {- None => Ok(None),- Some(v) => resolve_string(v),- }- })- .collect::<ArrowResult<StringArray>>()?,- )- as ArrayRef,- DataType::Binary | DataType::LargeBinary => Arc::new(- rows.iter()- .map(|row| {- let maybe_value = self.field_lookup(&field_path, row);- maybe_value.and_then(resolve_bytes)- })- .collect::<BinaryArray>(),- )- as ArrayRef,- DataType::FixedSizeBinary(size) => {- Arc::new(FixedSizeBinaryArray::try_from_sparse_iter_with_size(- rows.iter().map(|row| {- let maybe_value = self.field_lookup(&field_path, row);- maybe_value.and_then(|v| resolve_fixed(v, *size as usize))- }),- *size,- )?) as ArrayRef- }- DataType::List(list_field) => {- match list_field.data_type() {- DataType::Dictionary(key_ty, _) => {- self.build_wrapped_list_array(rows, &field_path, key_ty)?- }- _ => {- // extract rows by name- let extracted_rows = rows- .iter()- .map(|row| {- self.field_lookup(&field_path, row)- .unwrap_or(&Value::Null)- })- .collect::<Vec<&Value>>();- self.build_nested_list_array::<i32>(- &field_path,- &extracted_rows,- list_field,- )?- }- }- }- DataType::Dictionary(key_ty, val_ty) => self- .build_string_dictionary_array(- rows,- &field_path,- key_ty,- val_ty,- )?,- DataType::Struct(fields) => {- let len = rows.len();- let num_bytes = bit_util::ceil(len, 8);- let mut null_buffer = MutableBuffer::from_len_zeroed(num_bytes);- let empty_vec = vec![];- let struct_rows = rows- .iter()- .enumerate()- .map(|(i, row)| (i, self.field_lookup(&field_path, row)))- .map(|(i, v)| {- let v = v.map(maybe_resolve_union);- match v {- Some(Value::Record(value)) => {- bit_util::set_bit(&mut null_buffer, i);- value- }- None | Some(Value::Null) => &empty_vec,- other => {- panic!("expected struct got {other:?}");- }- }- })- .collect::<Vec<&Vec<(String, Value)>>>();- let arrays =- self.build_struct_array(&struct_rows, &field_path, fields)?;- // construct a struct array's data in order to set null buffer- let data_type = DataType::Struct(fields.clone());- let data = ArrayDataBuilder::new(data_type)- .len(len)- .null_bit_buffer(Some(null_buffer.into()))- .child_data(arrays.into_iter().map(|a| a.to_data()).collect())- .build()?;- make_array(data)- }- _ => {- return Err(SchemaError(format!(- "type {} not supported",- field.data_type()- )));- }- };- Ok(arr)- })- .collect();- arrays- }-- /// Read the primitive list's values into ArrayData- fn read_primitive_list_values<T>(&self, rows: &[&Value]) -> ArrayData- where- T: ArrowPrimitiveType + ArrowNumericType,- T::Native: NumCast,- {- let values = rows- .iter()- .flat_map(|row| {- let row = maybe_resolve_union(row);- if let Value::Array(values) = row {- values- .iter()- .map(resolve_item::<T>)- .collect::<Vec<Option<T::Native>>>()- } else if let Some(f) = resolve_item::<T>(row) {- vec![Some(f)]- } else {- vec![]- }- })- .collect::<Vec<Option<T::Native>>>();- let array = values.iter().collect::<PrimitiveArray<T>>();- array.to_data()- }-- fn field_lookup<'b>(- &self,- name: &str,- row: &'b [(String, Value)],- ) -> Option<&'b Value> {- self.schema_lookup- .get(name)- .and_then(|i| row.get(*i))- .map(|o| &o.1)- }-}--/// Flattens a list of Avro values, by flattening lists, and treating all other values as-/// single-value lists.-/// This is used to read into nested lists (list of list, list of struct) and non-dictionary lists.-#[inline]-fn flatten_values<'a>(values: &[&'a Value]) -> Vec<&'a Value> {- values- .iter()- .flat_map(|row| {- let v = maybe_resolve_union(row);- if let Value::Array(values) = v {- values.iter().collect()- } else {- // we interpret a scalar as a single-value list to minimise data loss- vec![v]- }- })- .collect()-}--/// Flattens a list into string values, dropping Value::Null in the process.-/// This is useful for interpreting any Avro array as string, dropping nulls.-/// See `value_as_string`.-#[inline]-fn flatten_string_values(values: &[&Value]) -> Vec<Option<String>> {- values- .iter()- .flat_map(|row| {- let row = maybe_resolve_union(row);- if let Value::Array(values) = row {- values- .iter()- .map(|s| resolve_string(s).ok().flatten())- .collect::<Vec<Option<_>>>()- } else if let Value::Null = row {- vec![]- } else {- vec![resolve_string(row).ok().flatten()]- }- })- .collect::<Vec<Option<_>>>()-}--/// Reads an Avro value as a string, regardless of its type.-/// This is useful if the expected datatype is a string, in which case we preserve-/// all the values regardless of they type.-fn resolve_string(v: &Value) -> ArrowResult<Option<String>> {- let v = if let Value::Union(_, b) = v { b } else { v };- match v {- Value::String(s) => Ok(Some(s.clone())),- Value::Bytes(bytes) => String::from_utf8(bytes.to_vec())- .map_err(|e| AvroError::new(AvroErrorDetails::ConvertToUtf8(e)))- .map(Some),- Value::Enum(_, s) => Ok(Some(s.clone())),- Value::Null => Ok(None),- other => Err(AvroError::new(AvroErrorDetails::GetString(other.clone()))),- }- .map_err(|e| SchemaError(format!("expected resolvable string : {e}")))-}--fn resolve_u8(v: &Value) -> Option<u8> {- let v = match v {- Value::Union(_, inner) => inner.as_ref(),- _ => v,- };-- match v {- Value::Int(n) => u8::try_from(*n).ok(),- Value::Long(n) => u8::try_from(*n).ok(),- _ => None,- }-}--fn resolve_bytes(v: &Value) -> Option<Vec<u8>> {- let v = match v {- Value::Union(_, inner) => inner.as_ref(),- _ => v,- };-- match v {- Value::Bytes(bytes) => Some(bytes.clone()),- Value::String(s) => Some(s.as_bytes().to_vec()),- Value::Array(items) => items.iter().map(resolve_u8).collect::<Option<Vec<u8>>>(),- _ => None,- }-}--fn resolve_fixed(v: &Value, size: usize) -> Option<Vec<u8>> {- let v = if let Value::Union(_, b) = v { b } else { v };- match v {- Value::Fixed(n, bytes) => {- if *n == size {- Some(bytes.clone())- } else {- None- }- }- _ => None,- }-}--fn resolve_boolean(value: &Value) -> Option<bool> {- let v = if let Value::Union(_, b) = value {- b- } else {- value- };- match v {- Value::Boolean(boolean) => Some(*boolean),- _ => None,- }-}--trait Resolver: ArrowPrimitiveType {- fn resolve(value: &Value) -> Option<Self::Native>;-}--fn resolve_item<T: Resolver>(value: &Value) -> Option<T::Native> {- T::resolve(value)-}--fn maybe_resolve_union(value: &Value) -> &Value {- if SchemaKind::from(value) == SchemaKind::Union {- // Pull out the Union, and attempt to resolve against it.- match value {- Value::Union(_, b) => b,- _ => unreachable!(),- }- } else {- value- }-}--impl<N> Resolver for N-where- N: ArrowNumericType,- N::Native: NumCast,-{- fn resolve(value: &Value) -> Option<Self::Native> {- let value = maybe_resolve_union(value);- match value {- Value::Int(i) | Value::TimeMillis(i) | Value::Date(i) => NumCast::from(*i),- Value::Long(l)- | Value::TimeMicros(l)- | Value::TimestampMillis(l)- | Value::TimestampMicros(l) => NumCast::from(*l),- Value::Float(f) => NumCast::from(*f),- Value::Double(f) => NumCast::from(*f),- Value::Duration(_d) => unimplemented!(), // shenanigans type- Value::Null => None,- _ => unreachable!(),- }- }-}--#[cfg(test)]-mod test {- use crate::avro_to_arrow::{Reader, ReaderBuilder};- use arrow::array::Array;- use arrow::datatypes::{DataType, Fields};- use arrow::datatypes::{Field, TimeUnit};- use datafusion_common::assert_batches_eq;- use datafusion_common::cast::{- as_int32_array, as_int64_array, as_list_array, as_timestamp_microsecond_array,- };- use std::fs::File;- use std::sync::Arc;-- fn build_reader(name: &'_ str, batch_size: usize) -> Reader<'_, File> {- let testdata = datafusion_common::test_util::arrow_test_data();- let filename = format!("{testdata}/avro/{name}");- let builder = ReaderBuilder::new()- .read_schema()- .with_batch_size(batch_size);- builder.build(File::open(filename).unwrap()).unwrap()- }-- // TODO: Fixed, Enum, Dictionary-- #[test]- fn test_time_avro_milliseconds() {- let mut reader = build_reader("alltypes_plain.avro", 10);- let batch = reader.next().unwrap().unwrap();-- assert_eq!(11, batch.num_columns());- assert_eq!(8, batch.num_rows());-- let schema = reader.schema();- let batch_schema = batch.schema();- assert_eq!(schema, batch_schema);-- let timestamp_col = schema.column_with_name("timestamp_col").unwrap();- assert_eq!(- &DataType::Timestamp(TimeUnit::Microsecond, None),- timestamp_col.1.data_type()- );- let timestamp_array =- as_timestamp_microsecond_array(batch.column(timestamp_col.0)).unwrap();- for i in 0..timestamp_array.len() {- assert!(timestamp_array.is_valid(i));- }- assert_eq!(1235865600000000, timestamp_array.value(0));- assert_eq!(1235865660000000, timestamp_array.value(1));- assert_eq!(1238544000000000, timestamp_array.value(2));- assert_eq!(1238544060000000, timestamp_array.value(3));- assert_eq!(1233446400000000, timestamp_array.value(4));- assert_eq!(1233446460000000, timestamp_array.value(5));- assert_eq!(1230768000000000, timestamp_array.value(6));- assert_eq!(1230768060000000, timestamp_array.value(7));- }-- #[test]- fn test_avro_read_list() {- let mut reader = build_reader("list_columns.avro", 3);- let schema = reader.schema();- let (col_id_index, _) = schema.column_with_name("int64_list").unwrap();- let batch = reader.next().unwrap().unwrap();- assert_eq!(batch.num_columns(), 2);- assert_eq!(batch.num_rows(), 3);- let a_array = as_list_array(batch.column(col_id_index)).unwrap();- assert_eq!(- *a_array.data_type(),- DataType::List(Arc::new(Field::new("element", DataType::Int64, true)))- );- let array = a_array.value(0);- assert_eq!(*array.data_type(), DataType::Int64);-- assert_eq!(- 6,- as_int64_array(&array)- .unwrap()- .iter()- .flatten()- .sum::<i64>()- );- }- #[test]- fn test_avro_read_nested_list() {- let mut reader = build_reader("nested_lists.snappy.avro", 3);- let batch = reader.next().unwrap().unwrap();- assert_eq!(batch.num_columns(), 2);- assert_eq!(batch.num_rows(), 3);- }-- #[test]- fn test_complex_list() {- let schema = apache_avro::Schema::parse_str(- r#"- {- "type": "record",- "name": "r1",- "fields": [- {- "name": "headers",- "type": ["null", {- "type": "array",- "items": ["null",{- "name":"r2",- "type": "record",- "fields":[- {"name":"name", "type": ["null", "string"], "default": null},- {"name":"value", "type": ["null", "string"], "default": null}- ]- }]- }],- "default": null- }- ]- }"#,- )- .unwrap();- let r1 = apache_avro::to_value(serde_json::json!({- "headers": [- {- "name": "a",- "value": "b"- }- ]- }))- .unwrap()- .resolve(&schema)- .unwrap();-- let mut w = apache_avro::Writer::new(&schema, vec![]);- w.append(r1).unwrap();- let bytes = w.into_inner().unwrap();-- let mut reader = ReaderBuilder::new()- .read_schema()- .with_batch_size(2)- .build(std::io::Cursor::new(bytes))- .unwrap();-- let batch = reader.next().unwrap().unwrap();- assert_eq!(batch.num_rows(), 1);- assert_eq!(batch.num_columns(), 1);- let expected = [- "+-----------------------+",- "| headers |",- "+-----------------------+",- "| [{name: a, value: b}] |",- "+-----------------------+",- ];- assert_batches_eq!(expected, &[batch]);- }-- #[test]- fn test_complex_struct() {- let schema = apache_avro::Schema::parse_str(- r#"- {- "type": "record",- "name": "r1",- "fields": [- {- "name": "dns",- "type": [- "null",- {- "type": "record",- "name": "r13",- "fields": [- {- "name": "answers",- "type": [- "null",- {- "type": "array",- "items": [- "null",- {- "type": "record",- "name": "r292",- "fields": [- {- "name": "class",- "type": ["null", "string"],- "default": null- },- {- "name": "data",- "type": ["null", "string"],- "default": null- },- {- "name": "name",- "type": ["null", "string"],- "default": null- },- {- "name": "ttl",- "type": ["null", "long"],- "default": null- },- {- "name": "type",- "type": ["null", "string"],- "default": null- }- ]- }- ]- }- ],- "default": null- },- {- "name": "header_flags",- "type": [- "null",- {- "type": "array",- "items": ["null", "string"]- }- ],- "default": null- },- {- "name": "id",- "type": ["null", "string"],- "default": null- },- {- "name": "op_code",- "type": ["null", "string"],- "default": null- },- {- "name": "question",- "type": [- "null",- {- "type": "record",- "name": "r288",- "fields": [- {- "name": "class",- "type": ["null", "string"],- "default": null- },- {- "name": "name",- "type": ["null", "string"],- "default": null- },- {- "name": "registered_domain",- "type": ["null", "string"],- "default": null- },- {- "name": "subdomain",- "type": ["null", "string"],- "default": null- },- {- "name": "top_level_domain",- "type": ["null", "string"],- "default": null- },- {- "name": "type",- "type": ["null", "string"],- "default": null- }- ]- }- ],- "default": null- },- {- "name": "resolved_ip",- "type": [- "null",- {- "type": "array",- "items": ["null", "string"]- }- ],- "default": null- },- {- "name": "response_code",- "type": ["null", "string"],- "default": null- },- {- "name": "type",- "type": ["null", "string"],- "default": null- }- ]- }- ],- "default": null- }- ]- }"#,- )- .unwrap();-- let jv1 = serde_json::json!({- "dns": {- "answers": [- {- "data": "CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=",- "type": "1"- },- {- "data": "CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=",- "type": "1"- },- {- "data": "CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=",- "type": "1"- }- ],- "question": {- "name": "security.ubuntu.com",- "type": "A"- },- "resolved_ip": [- "67.43.156.1",- "67.43.156.2",- "67.43.156.3"- ],- "response_code": "0"- }- });- let r1 = apache_avro::to_value(jv1)- .unwrap()- .resolve(&schema)- .unwrap();-- let mut w = apache_avro::Writer::new(&schema, vec![]);- w.append(r1).unwrap();- let bytes = w.into_inner().unwrap();-- let mut reader = ReaderBuilder::new()- .read_schema()- .with_batch_size(1)- .build(std::io::Cursor::new(bytes))- .unwrap();-- let batch = reader.next().unwrap().unwrap();- assert_eq!(batch.num_rows(), 1);- assert_eq!(batch.num_columns(), 1);-- let expected = [- "+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+",- "| dns |",- "+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+",- "| {answers: [{class: , data: CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=, name: , ttl: , type: 1}, {class: , data: CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=, name: , ttl: , type: 1}, {class: , data: CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=, name: , ttl: , type: 1}], header_flags: , id: , op_code: , question: {class: , name: security.ubuntu.com, registered_domain: , subdomain: , top_level_domain: , type: A}, resolved_ip: [67.43.156.1, 67.43.156.2, 67.43.156.3], response_code: 0, type: } |",- "+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+",- ];- assert_batches_eq!(expected, &[batch]);- }-- #[test]- fn test_deep_nullable_struct() {- let schema = apache_avro::Schema::parse_str(- r#"- {- "type": "record",- "name": "r1",- "fields": [- {- "name": "col1",- "type": [- "null",- {- "type": "record",- "name": "r2",- "fields": [- {- "name": "col2",- "type": [- "null",- {- "type": "record",- "name": "r3",- "fields": [- {- "name": "col3",- "type": [- "null",- {- "type": "record",- "name": "r4",- "fields": [- {- "name": "col4",- "type": [- "null",- {- "type": "record",- "name": "r5",- "fields": [- {- "name": "col5",- "type": ["null", "string"]- }- ]- }- ]- }- ]- }- ]- }- ]- }- ]- }- ]- }- ]- }- ]- }- "#,- )- .unwrap();- let r1 = apache_avro::to_value(serde_json::json!({- "col1": {- "col2": {- "col3": {- "col4": {- "col5": "hello"- }- }- }- }- }))- .unwrap()- .resolve(&schema)- .unwrap();- let r2 = apache_avro::to_value(serde_json::json!({- "col1": {- "col2": {- "col3": {- "col4": {- "col5": null- }- }- }- }- }))- .unwrap()- .resolve(&schema)- .unwrap();- let r3 = apache_avro::to_value(serde_json::json!({- "col1": {- "col2": {- "col3": null- }- }- }))- .unwrap()- .resolve(&schema)- .unwrap();- let r4 = apache_avro::to_value(serde_json::json!({ "col1": null }))- .unwrap()- .resolve(&schema)- .unwrap();-- let mut w = apache_avro::Writer::new(&schema, vec![]);- w.append(r1).unwrap();- w.append(r2).unwrap();- w.append(r3).unwrap();- w.append(r4).unwrap();- let bytes = w.into_inner().unwrap();-- let mut reader = ReaderBuilder::new()- .read_schema()- .with_batch_size(4)- .build(std::io::Cursor::new(bytes))- .unwrap();-- let batch = reader.next().unwrap().unwrap();-- let expected = [- "+---------------------------------------+",- "| col1 |",- "+---------------------------------------+",- "| {col2: {col3: {col4: {col5: hello}}}} |",- "| {col2: {col3: {col4: {col5: }}}} |",- "| {col2: {col3: }} |",- "| |",- "+---------------------------------------+",- ];- assert_batches_eq!(expected, &[batch]);- }-- #[test]- fn test_avro_nullable_struct() {- let schema = apache_avro::Schema::parse_str(- r#"- {- "type": "record",- "name": "r1",- "fields": [- {- "name": "col1",- "type": [- "null",- {- "type": "record",- "name": "r2",- "fields": [- {- "name": "col2",- "type": ["null", "string"]- }- ]- }- ],- "default": null- }- ]- }"#,- )- .unwrap();- let r1 = apache_avro::to_value(serde_json::json!({ "col1": null }))- .unwrap()- .resolve(&schema)- .unwrap();- let r2 = apache_avro::to_value(serde_json::json!({- "col1": {- "col2": "hello"- }- }))- .unwrap()- .resolve(&schema)- .unwrap();- let r3 = apache_avro::to_value(serde_json::json!({- "col1": {- "col2": null- }- }))- .unwrap()- .resolve(&schema)- .unwrap();-- let mut w = apache_avro::Writer::new(&schema, vec![]);- w.append(r1).unwrap();- w.append(r2).unwrap();- w.append(r3).unwrap();- let bytes = w.into_inner().unwrap();-- let mut reader = ReaderBuilder::new()- .read_schema()- .with_batch_size(3)- .build(std::io::Cursor::new(bytes))- .unwrap();- let batch = reader.next().unwrap().unwrap();- assert_eq!(batch.num_rows(), 3);- assert_eq!(batch.num_columns(), 1);-- let expected = [- "+---------------+",- "| col1 |",- "+---------------+",- "| |",- "| {col2: hello} |",- "| {col2: } |",- "+---------------+",- ];- assert_batches_eq!(expected, &[batch]);- }-- #[test]- fn test_avro_nullable_struct_array() {- let schema = apache_avro::Schema::parse_str(- r#"- {- "type": "record",- "name": "r1",- "fields": [- {- "name": "col1",- "type": [- "null",- {- "type": "array",- "items": {- "type": [- "null",- {- "type": "record",- "name": "Item",- "fields": [- {- "name": "id",- "type": "long"- }- ]- }- ]- }- }- ],- "default": null- }- ]- }"#,- )- .unwrap();- let jv1 = serde_json::json!({- "col1": [- {- "id": 234- },- {- "id": 345- }- ]- });- let r1 = apache_avro::to_value(jv1)- .unwrap()- .resolve(&schema)- .unwrap();- let r2 = apache_avro::to_value(serde_json::json!({ "col1": null }))- .unwrap()- .resolve(&schema)- .unwrap();-- let mut w = apache_avro::Writer::new(&schema, vec![]);- for _i in 0..5 {- w.append(r1.clone()).unwrap();- }- w.append(r2).unwrap();- let bytes = w.into_inner().unwrap();-- let mut reader = ReaderBuilder::new()- .read_schema()- .with_batch_size(20)- .build(std::io::Cursor::new(bytes))- .unwrap();- let batch = reader.next().unwrap().unwrap();- assert_eq!(batch.num_rows(), 6);- assert_eq!(batch.num_columns(), 1);-- let expected = [- "+------------------------+",- "| col1 |",- "+------------------------+",- "| [{id: 234}, {id: 345}] |",- "| [{id: 234}, {id: 345}] |",- "| [{id: 234}, {id: 345}] |",- "| [{id: 234}, {id: 345}] |",- "| [{id: 234}, {id: 345}] |",- "| |",- "+------------------------+",- ];- assert_batches_eq!(expected, &[batch]);- }-- #[test]- fn test_avro_iterator() {- let reader = build_reader("alltypes_plain.avro", 5);- let schema = reader.schema();- let (col_id_index, _) = schema.column_with_name("id").unwrap();-- let mut sum_num_rows = 0;- let mut num_batches = 0;- let mut sum_id = 0;- for batch in reader {- let batch = batch.unwrap();- assert_eq!(11, batch.num_columns());- sum_num_rows += batch.num_rows();- num_batches += 1;- let batch_schema = batch.schema();- assert_eq!(schema, batch_schema);- let a_array = as_int32_array(batch.column(col_id_index)).unwrap();- sum_id += (0..a_array.len()).map(|i| a_array.value(i)).sum::<i32>();- }- assert_eq!(8, sum_num_rows);- assert_eq!(2, num_batches);- assert_eq!(28, sum_id);- }-- #[test]- fn test_list_of_structs_with_custom_field_name() {- let schema = apache_avro::Schema::parse_str(- r#"- {- "type": "record",- "name": "root",- "fields": [- {- "name": "items",- "type": {- "type": "array",- "items": {- "type": "record",- "name": "item_record",- "fields": [- {- "name": "id",- "type": "long"- },- {- "name": "name",- "type": "string"- }- ]- }- }- }- ]- }"#,- )- .unwrap();-- let r1 = apache_avro::to_value(serde_json::json!({- "items": [- {- "id": 1,- "name": "first"- },- {- "id": 2,- "name": "second"- }- ]- }))- .unwrap()- .resolve(&schema)- .unwrap();-- let mut w = apache_avro::Writer::new(&schema, vec![]);- w.append(r1).unwrap();- let bytes = w.into_inner().unwrap();-- // Create an Arrow schema where the list field is NOT named "element"- let arrow_schema = Arc::new(arrow::datatypes::Schema::new(vec![Field::new(- "items",- DataType::List(Arc::new(Field::new(- "item", // This is NOT "element"- DataType::Struct(Fields::from(vec![- Field::new("id", DataType::Int64, false),- Field::new("name", DataType::Utf8, false),- ])),- false,- ))),- false,- )]));-- let mut reader = ReaderBuilder::new()- .with_schema(arrow_schema)- .with_batch_size(10)- .build(std::io::Cursor::new(bytes))- .unwrap();-- // This used to fail because schema_lookup would have "items.element.id" and "items.element.name"- // but build_struct_array will try to look up "items.item.id" and "items.item.name",- // Now it it is simply "items.id" and "items.name"- let batch = reader.next().unwrap().unwrap();-- let expected = [- "+-----------------------------------------------+",- "| items |",- "+-----------------------------------------------+",- "| [{id: 1, name: first}, {id: 2, name: second}] |",- "+-----------------------------------------------+",- ];- assert_batches_eq!(expected, &[batch]);- }-}diff --git a/datafusion/datasource-avro/src/avro_to_arrow/mod.rs b/datafusion/datasource-avro/src/avro_to_arrow/mod.rsdeleted file mode 100644index c1530a4880..0000000000--- a/datafusion/datasource-avro/src/avro_to_arrow/mod.rs+++ /dev/null@@ -1,39 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--//! This module contains code for reading [Avro] data into `RecordBatch`es-//!-//! [Avro]: https://avro.apache.org/docs/1.2.0/--mod arrow_array_reader;-mod reader;-mod schema;--use arrow::datatypes::Schema;-pub use reader::{Reader, ReaderBuilder};--pub use schema::to_arrow_schema;-use std::io::Read;--/// Read Avro schema given a reader-pub fn read_avro_schema_from_reader<R: Read>(- reader: &mut R,-) -> datafusion_common::Result<Schema> {- let avro_reader = apache_avro::Reader::new(reader)?;- let schema = avro_reader.writer_schema();- to_arrow_schema(schema)-}diff --git a/datafusion/datasource-avro/src/avro_to_arrow/reader.rs b/datafusion/datasource-avro/src/avro_to_arrow/reader.rsdeleted file mode 100644index bd96b47aea..0000000000--- a/datafusion/datasource-avro/src/avro_to_arrow/reader.rs+++ /dev/null@@ -1,353 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--use super::arrow_array_reader::AvroArrowArrayReader;-use arrow::datatypes::{Fields, SchemaRef};-use arrow::error::Result as ArrowResult;-use arrow::record_batch::RecordBatch;-use datafusion_common::Result;-use std::io::{Read, Seek};-use std::sync::Arc;--/// Avro file reader builder-#[derive(Debug)]-pub struct ReaderBuilder {- /// Optional schema for the Avro file- ///- /// If the schema is not supplied, the reader will try to read the schema.- schema: Option<SchemaRef>,- /// Batch size (number of records to load each time)- ///- /// The default batch size when using the `ReaderBuilder` is 1024 records- batch_size: usize,- /// Optional projection for which columns to load (zero-based column indices)- projection: Option<Vec<String>>,-}--impl Default for ReaderBuilder {- fn default() -> Self {- Self {- schema: None,- batch_size: 1024,- projection: None,- }- }-}--impl ReaderBuilder {- /// Create a new builder for configuring Avro parsing options.- ///- /// To convert a builder into a reader, call `Reader::from_builder`- ///- /// # Example- ///- /// ```- /// use std::fs::File;- ///- /// use datafusion_datasource_avro::avro_to_arrow::{Reader, ReaderBuilder};- ///- /// fn example() -> Reader<'static, File> {- /// let file = File::open("test/data/basic.avro").unwrap();- ///- /// // create a builder, inferring the schema with the first 100 records- /// let builder = ReaderBuilder::new().read_schema().with_batch_size(100);- ///- /// let reader = builder.build::<File>(file).unwrap();- ///- /// reader- /// }- /// ```- pub fn new() -> Self {- Self::default()- }-- /// Set the Avro file's schema- pub fn with_schema(mut self, schema: SchemaRef) -> Self {- self.schema = Some(schema);- self- }-- /// Set the Avro reader to infer the schema of the file- pub fn read_schema(mut self) -> Self {- // remove any schema that is set- self.schema = None;- self- }-- /// Set the batch size (number of records to load at one time)- pub fn with_batch_size(mut self, batch_size: usize) -> Self {- self.batch_size = batch_size;- self- }-- /// Set the reader's column projection- pub fn with_projection(mut self, projection: Vec<String>) -> Self {- self.projection = Some(projection);- self- }-- /// Create a new `Reader` from the `ReaderBuilder`- pub fn build<'a, R>(self, source: R) -> Result<Reader<'a, R>>- where- R: Read + Seek,- {- let mut source = source;-- // check if schema should be inferred- let schema = match self.schema {- Some(schema) => schema,- None => Arc::new(super::read_avro_schema_from_reader(&mut source)?),- };- source.rewind()?;- Reader::try_new(source, &schema, self.batch_size, self.projection.as_ref())- }-}--/// Avro file record reader-pub struct Reader<'a, R: Read> {- array_reader: AvroArrowArrayReader<'a, R>,- schema: SchemaRef,- batch_size: usize,-}--impl<R: Read> Reader<'_, R> {- /// Create a new Avro Reader from any value that implements the `Read` trait.- ///- /// If reading a `File`, you can customise the Reader, such as to enable schema- /// inference, use `ReaderBuilder`.- ///- /// If projection is provided, it uses a schema with only the fields in the projection, respecting their order.- /// Only the first level of projection is handled. No further projection currently occurs, but would be- /// useful if plucking values from a struct, e.g. getting `a.b.c.e` from `a.b.c.{d, e}`.- pub fn try_new(- reader: R,- schema: &SchemaRef,- batch_size: usize,- projection: Option<&Vec<String>>,- ) -> Result<Self> {- let projected_schema = projection.as_ref().filter(|p| !p.is_empty()).map_or_else(- || Arc::clone(schema),- |proj| {- Arc::new(arrow::datatypes::Schema::new(- proj.iter()- .filter_map(|name| {- schema.column_with_name(name).map(|(_, f)| f.clone())- })- .collect::<Fields>(),- ))- },- );-- Ok(Self {- array_reader: AvroArrowArrayReader::try_new(- reader,- Arc::clone(&projected_schema),- )?,- schema: projected_schema,- batch_size,- })- }-- /// Returns the schema of the reader, useful for getting the schema without reading- /// record batches- pub fn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)- }-}--impl<R: Read> Iterator for Reader<'_, R> {- type Item = ArrowResult<RecordBatch>;-- /// Returns the next batch of results (defined by `self.batch_size`), or `None` if there- /// are no more results.- fn next(&mut self) -> Option<Self::Item> {- self.array_reader.next_batch(self.batch_size)- }-}--#[cfg(test)]-mod tests {- use super::*;- use arrow::array::*;- use arrow::array::{- BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array,- TimestampMicrosecondArray,- };- use arrow::datatypes::TimeUnit;- use arrow::datatypes::{DataType, Field};- use std::fs::File;-- fn build_reader(name: &'_ str, projection: Option<Vec<String>>) -> Reader<'_, File> {- let testdata = datafusion_common::test_util::arrow_test_data();- let filename = format!("{testdata}/avro/{name}");- let mut builder = ReaderBuilder::new().read_schema().with_batch_size(64);- if let Some(projection) = projection {- builder = builder.with_projection(projection);- }- builder.build(File::open(filename).unwrap()).unwrap()- }-- fn get_col<'a, T: 'static>(- batch: &'a RecordBatch,- col: (usize, &Field),- ) -> Option<&'a T> {- batch.column(col.0).as_any().downcast_ref::<T>()- }-- #[test]- fn test_avro_basic() {- let mut reader = build_reader("alltypes_dictionary.avro", None);- let batch = reader.next().unwrap().unwrap();-- assert_eq!(11, batch.num_columns());- assert_eq!(2, batch.num_rows());-- let schema = reader.schema();- let batch_schema = batch.schema();- assert_eq!(schema, batch_schema);-- let id = schema.column_with_name("id").unwrap();- assert_eq!(0, id.0);- assert_eq!(&DataType::Int32, id.1.data_type());- let col = get_col::<Int32Array>(&batch, id).unwrap();- assert_eq!(0, col.value(0));- assert_eq!(1, col.value(1));- let bool_col = schema.column_with_name("bool_col").unwrap();- assert_eq!(1, bool_col.0);- assert_eq!(&DataType::Boolean, bool_col.1.data_type());- let col = get_col::<BooleanArray>(&batch, bool_col).unwrap();- assert!(col.value(0));- assert!(!col.value(1));- let tinyint_col = schema.column_with_name("tinyint_col").unwrap();- assert_eq!(2, tinyint_col.0);- assert_eq!(&DataType::Int32, tinyint_col.1.data_type());- let col = get_col::<Int32Array>(&batch, tinyint_col).unwrap();- assert_eq!(0, col.value(0));- assert_eq!(1, col.value(1));- let smallint_col = schema.column_with_name("smallint_col").unwrap();- assert_eq!(3, smallint_col.0);- assert_eq!(&DataType::Int32, smallint_col.1.data_type());- let col = get_col::<Int32Array>(&batch, smallint_col).unwrap();- assert_eq!(0, col.value(0));- assert_eq!(1, col.value(1));- let int_col = schema.column_with_name("int_col").unwrap();- assert_eq!(4, int_col.0);- let col = get_col::<Int32Array>(&batch, int_col).unwrap();- assert_eq!(0, col.value(0));- assert_eq!(1, col.value(1));- assert_eq!(&DataType::Int32, int_col.1.data_type());- let col = get_col::<Int32Array>(&batch, int_col).unwrap();- assert_eq!(0, col.value(0));- assert_eq!(1, col.value(1));- let bigint_col = schema.column_with_name("bigint_col").unwrap();- assert_eq!(5, bigint_col.0);- let col = get_col::<Int64Array>(&batch, bigint_col).unwrap();- assert_eq!(0, col.value(0));- assert_eq!(10, col.value(1));- assert_eq!(&DataType::Int64, bigint_col.1.data_type());- let float_col = schema.column_with_name("float_col").unwrap();- assert_eq!(6, float_col.0);- let col = get_col::<Float32Array>(&batch, float_col).unwrap();- assert_eq!(0.0, col.value(0));- assert_eq!(1.1, col.value(1));- assert_eq!(&DataType::Float32, float_col.1.data_type());- let col = get_col::<Float32Array>(&batch, float_col).unwrap();- assert_eq!(0.0, col.value(0));- assert_eq!(1.1, col.value(1));- let double_col = schema.column_with_name("double_col").unwrap();- assert_eq!(7, double_col.0);- assert_eq!(&DataType::Float64, double_col.1.data_type());- let col = get_col::<Float64Array>(&batch, double_col).unwrap();- assert_eq!(0.0, col.value(0));- assert_eq!(10.1, col.value(1));- let date_string_col = schema.column_with_name("date_string_col").unwrap();- assert_eq!(8, date_string_col.0);- assert_eq!(&DataType::Binary, date_string_col.1.data_type());- let col = get_col::<BinaryArray>(&batch, date_string_col).unwrap();- assert_eq!("01/01/09".as_bytes(), col.value(0));- assert_eq!("01/01/09".as_bytes(), col.value(1));- let string_col = schema.column_with_name("string_col").unwrap();- assert_eq!(9, string_col.0);- assert_eq!(&DataType::Binary, string_col.1.data_type());- let col = get_col::<BinaryArray>(&batch, string_col).unwrap();- assert_eq!("0".as_bytes(), col.value(0));- assert_eq!("1".as_bytes(), col.value(1));- let timestamp_col = schema.column_with_name("timestamp_col").unwrap();- assert_eq!(10, timestamp_col.0);- assert_eq!(- &DataType::Timestamp(TimeUnit::Microsecond, None),- timestamp_col.1.data_type()- );- let col = get_col::<TimestampMicrosecondArray>(&batch, timestamp_col).unwrap();- assert_eq!(1230768000000000, col.value(0));- assert_eq!(1230768060000000, col.value(1));- }-- #[test]- fn test_avro_with_projection() {- // Test projection to filter and reorder columns- let projection = Some(vec![- "string_col".to_string(),- "double_col".to_string(),- "bool_col".to_string(),- ]);- let mut reader = build_reader("alltypes_dictionary.avro", projection);- let batch = reader.next().unwrap().unwrap();-- // Only 3 columns should be present (not all 11)- assert_eq!(3, batch.num_columns());- assert_eq!(2, batch.num_rows());-- let schema = reader.schema();- let batch_schema = batch.schema();- assert_eq!(schema, batch_schema);-- // Verify columns are in the order specified in projection- // First column should be string_col (was at index 9 in original)- assert_eq!("string_col", schema.field(0).name());- assert_eq!(&DataType::Binary, schema.field(0).data_type());- let col = batch- .column(0)- .as_any()- .downcast_ref::<BinaryArray>()- .unwrap();- assert_eq!("0".as_bytes(), col.value(0));- assert_eq!("1".as_bytes(), col.value(1));-- // Second column should be double_col (was at index 7 in original)- assert_eq!("double_col", schema.field(1).name());- assert_eq!(&DataType::Float64, schema.field(1).data_type());- let col = batch- .column(1)- .as_any()- .downcast_ref::<Float64Array>()- .unwrap();- assert_eq!(0.0, col.value(0));- assert_eq!(10.1, col.value(1));-- // Third column should be bool_col (was at index 1 in original)- assert_eq!("bool_col", schema.field(2).name());- assert_eq!(&DataType::Boolean, schema.field(2).data_type());- let col = batch- .column(2)- .as_any()- .downcast_ref::<BooleanArray>()- .unwrap();- assert!(col.value(0));- assert!(!col.value(1));- }-}diff --git a/datafusion/datasource-avro/src/avro_to_arrow/schema.rs b/datafusion/datasource-avro/src/avro_to_arrow/schema.rsdeleted file mode 100644index 053be3c9af..0000000000--- a/datafusion/datasource-avro/src/avro_to_arrow/schema.rs+++ /dev/null@@ -1,517 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--use apache_avro::Schema as AvroSchema;-use apache_avro::schema::{- Alias, DecimalSchema, EnumSchema, FixedSchema, Name, RecordSchema,-};-use apache_avro::types::Value;-use arrow::datatypes::{DataType, IntervalUnit, Schema, TimeUnit, UnionMode};-use arrow::datatypes::{Field, UnionFields};-use datafusion_common::error::Result;-use std::collections::HashMap;-use std::sync::Arc;--/// Converts an avro schema to an arrow schema-pub fn to_arrow_schema(avro_schema: &apache_avro::Schema) -> Result<Schema> {- let mut schema_fields = vec![];- match avro_schema {- AvroSchema::Record(RecordSchema { fields, .. }) => {- for field in fields {- schema_fields.push(schema_to_field_with_props(- &field.schema,- Some(&field.name),- field.is_nullable(),- Some(external_props(&field.schema)),- )?)- }- }- schema => schema_fields.push(schema_to_field(schema, Some(""), false)?),- }-- let schema = Schema::new(schema_fields);- Ok(schema)-}--fn schema_to_field(- schema: &apache_avro::Schema,- name: Option<&str>,- nullable: bool,-) -> Result<Field> {- schema_to_field_with_props(schema, name, nullable, Default::default())-}--fn schema_to_field_with_props(- schema: &AvroSchema,- name: Option<&str>,- nullable: bool,- props: Option<HashMap<String, String>>,-) -> Result<Field> {- let mut nullable = nullable;- let field_type: DataType = match schema {- AvroSchema::Ref { .. } => todo!("Add support for AvroSchema::Ref"),- AvroSchema::Null => DataType::Null,- AvroSchema::Boolean => DataType::Boolean,- AvroSchema::Int => DataType::Int32,- AvroSchema::Long => DataType::Int64,- AvroSchema::Float => DataType::Float32,- AvroSchema::Double => DataType::Float64,- AvroSchema::Bytes => DataType::Binary,- AvroSchema::String => DataType::Utf8,- AvroSchema::Array(item_schema) => DataType::List(Arc::new(- schema_to_field_with_props(&item_schema.items, Some("element"), false, None)?,- )),- AvroSchema::Map(value_schema) => {- let value_field = schema_to_field_with_props(- &value_schema.types,- Some("value"),- false,- None,- )?;- DataType::Dictionary(- Box::new(DataType::Utf8),- Box::new(value_field.data_type().clone()),- )- }- AvroSchema::Union(us) => {- // If there are only two variants and one of them is null, set the other type as the field data type- let has_nullable = us- .find_schema_with_known_schemata::<apache_avro::Schema>(- &Value::Null,- None,- &None,- )- .is_some();- let sub_schemas = us.variants();- if has_nullable && sub_schemas.len() == 2 {- nullable = true;- if let Some(schema) = sub_schemas- .iter()- .find(|&schema| !matches!(schema, AvroSchema::Null))- {- schema_to_field_with_props(schema, None, has_nullable, None)?- .data_type()- .clone()- } else {- return Err(apache_avro::Error::new(- apache_avro::error::Details::GetUnionDuplicate,- )- .into());- }- } else {- let fields = sub_schemas- .iter()- .map(|s| schema_to_field_with_props(s, None, has_nullable, None))- .collect::<Result<Vec<Field>>>()?;- // Assign type_ids based on the order in which they appear- DataType::Union(UnionFields::from_fields(fields), UnionMode::Dense)- }- }- AvroSchema::Record(RecordSchema { fields, .. }) => {- let fields: Result<_> = fields- .iter()- .map(|field| {- let mut props = HashMap::new();- if let Some(doc) = &field.doc {- props.insert("avro::doc".to_string(), doc.clone());- }- /*if let Some(aliases) = fields.aliases {- props.insert("aliases", aliases);- }*/- schema_to_field_with_props(- &field.schema,- Some(&field.name),- false,- Some(props),- )- })- .collect();- DataType::Struct(fields?)- }- AvroSchema::Enum(EnumSchema { .. }) => DataType::Utf8,- AvroSchema::Fixed(FixedSchema { size, .. }) => {- DataType::FixedSizeBinary(*size as i32)- }- AvroSchema::Decimal(DecimalSchema {- precision, scale, ..- }) => DataType::Decimal128(*precision as u8, *scale as i8),- AvroSchema::BigDecimal => DataType::LargeBinary,- AvroSchema::Uuid => DataType::FixedSizeBinary(16),- AvroSchema::Date => DataType::Date32,- AvroSchema::TimeMillis => DataType::Time32(TimeUnit::Millisecond),- AvroSchema::TimeMicros => DataType::Time64(TimeUnit::Microsecond),- AvroSchema::TimestampMillis => DataType::Timestamp(TimeUnit::Millisecond, None),- AvroSchema::TimestampMicros => DataType::Timestamp(TimeUnit::Microsecond, None),- AvroSchema::TimestampNanos => DataType::Timestamp(TimeUnit::Nanosecond, None),- AvroSchema::LocalTimestampMillis => todo!(),- AvroSchema::LocalTimestampMicros => todo!(),- AvroSchema::LocalTimestampNanos => todo!(),- AvroSchema::Duration => DataType::Duration(TimeUnit::Millisecond),- };-- let data_type = field_type.clone();- let name = name.unwrap_or_else(|| default_field_name(&data_type));-- let mut field = Field::new(name, field_type, nullable);- field.set_metadata(props.unwrap_or_default());- Ok(field)-}--fn default_field_name(dt: &DataType) -> &str {- match dt {- DataType::Null => "null",- DataType::Boolean => "bit",- DataType::Int8 => "tinyint",- DataType::Int16 => "smallint",- DataType::Int32 => "int",- DataType::Int64 => "bigint",- DataType::UInt8 => "uint1",- DataType::UInt16 => "uint2",- DataType::UInt32 => "uint4",- DataType::UInt64 => "uint8",- DataType::Float16 => "float2",- DataType::Float32 => "float4",- DataType::Float64 => "float8",- DataType::Date32 => "dateday",- DataType::Date64 => "datemilli",- DataType::Time32(tu) | DataType::Time64(tu) => match tu {- TimeUnit::Second => "timesec",- TimeUnit::Millisecond => "timemilli",- TimeUnit::Microsecond => "timemicro",- TimeUnit::Nanosecond => "timenano",- },- DataType::Timestamp(tu, tz) => {- if tz.is_some() {- match tu {- TimeUnit::Second => "timestampsectz",- TimeUnit::Millisecond => "timestampmillitz",- TimeUnit::Microsecond => "timestampmicrotz",- TimeUnit::Nanosecond => "timestampnanotz",- }- } else {- match tu {- TimeUnit::Second => "timestampsec",- TimeUnit::Millisecond => "timestampmilli",- TimeUnit::Microsecond => "timestampmicro",- TimeUnit::Nanosecond => "timestampnano",- }- }- }- DataType::Duration(_) => "duration",- DataType::Interval(unit) => match unit {- IntervalUnit::YearMonth => "intervalyear",- IntervalUnit::DayTime => "intervalmonth",- IntervalUnit::MonthDayNano => "intervalmonthdaynano",- },- DataType::Binary => "varbinary",- DataType::FixedSizeBinary(_) => "fixedsizebinary",- DataType::LargeBinary => "largevarbinary",- DataType::Utf8 => "varchar",- DataType::LargeUtf8 => "largevarchar",- DataType::List(_) => "list",- DataType::FixedSizeList(_, _) => "fixed_size_list",- DataType::LargeList(_) => "largelist",- DataType::Struct(_) => "struct",- DataType::Union(_, _) => "union",- DataType::Dictionary(_, _) => "map",- DataType::Map(_, _) => unimplemented!("Map support not implemented"),- DataType::RunEndEncoded(_, _) => {- unimplemented!("RunEndEncoded support not implemented")- }- DataType::Utf8View- | DataType::BinaryView- | DataType::ListView(_)- | DataType::LargeListView(_) => {- unimplemented!("View support not implemented")- }- DataType::Decimal32(_, _) => "decimal",- DataType::Decimal64(_, _) => "decimal",- DataType::Decimal128(_, _) => "decimal",- DataType::Decimal256(_, _) => "decimal",- }-}--fn external_props(schema: &AvroSchema) -> HashMap<String, String> {- let mut props = HashMap::new();- match &schema {- AvroSchema::Record(RecordSchema { doc: Some(doc), .. })- | AvroSchema::Enum(EnumSchema { doc: Some(doc), .. })- | AvroSchema::Fixed(FixedSchema { doc: Some(doc), .. }) => {- props.insert("avro::doc".to_string(), doc.clone());- }- _ => {}- }- match &schema {- AvroSchema::Record(RecordSchema {- name: Name { namespace, .. },- aliases: Some(aliases),- ..- })- | AvroSchema::Enum(EnumSchema {- name: Name { namespace, .. },- aliases: Some(aliases),- ..- })- | AvroSchema::Fixed(FixedSchema {- name: Name { namespace, .. },- aliases: Some(aliases),- ..- }) => {- let aliases: Vec<String> = aliases- .iter()- .map(|alias| aliased(alias, namespace.as_deref(), None))- .collect();- props.insert(- "avro::aliases".to_string(),- format!("[{}]", aliases.join(",")),- );- }- _ => {}- }- props-}--/// Returns the fully qualified name for a field-pub fn aliased(- alias: &Alias,- namespace: Option<&str>,- default_namespace: Option<&str>,-) -> String {- if alias.namespace().is_some() {- alias.fullname(None)- } else {- let namespace = namespace.as_ref().copied().or(default_namespace);-- match namespace {- Some(ref namespace) => format!("{}.{}", namespace, alias.name()),- None => alias.fullname(None),- }- }-}--#[cfg(test)]-mod test {- use super::{aliased, external_props, to_arrow_schema};- use apache_avro::Schema as AvroSchema;- use apache_avro::schema::{Alias, EnumSchema, FixedSchema, Name, RecordSchema};- use arrow::datatypes::DataType::{Binary, Float32, Float64, Timestamp, Utf8};- use arrow::datatypes::DataType::{Boolean, Int32, Int64};- use arrow::datatypes::TimeUnit::Microsecond;- use arrow::datatypes::{Field, Schema};-- fn alias(name: &str) -> Alias {- Alias::new(name).unwrap()- }-- #[test]- fn test_alias() {- assert_eq!(aliased(&alias("foo.bar"), None, None), "foo.bar");- assert_eq!(aliased(&alias("bar"), Some("foo"), None), "foo.bar");- assert_eq!(aliased(&alias("bar"), Some("foo"), Some("cat")), "foo.bar");- assert_eq!(aliased(&alias("bar"), None, Some("cat")), "cat.bar");- }-- #[test]- fn test_external_props() {- let record_schema = AvroSchema::Record(RecordSchema {- name: Name {- name: "record".to_string(),- namespace: None,- },- aliases: Some(vec![alias("fooalias"), alias("baralias")]),- doc: Some("record documentation".to_string()),- fields: vec![],- lookup: Default::default(),- attributes: Default::default(),- });- let props = external_props(&record_schema);- assert_eq!(- props.get("avro::doc"),- Some(&"record documentation".to_string())- );- assert_eq!(- props.get("avro::aliases"),- Some(&"[fooalias,baralias]".to_string())- );- let enum_schema = AvroSchema::Enum(EnumSchema {- name: Name {- name: "enum".to_string(),- namespace: None,- },- aliases: Some(vec![alias("fooenum"), alias("barenum")]),- doc: Some("enum documentation".to_string()),- symbols: vec![],- default: None,- attributes: Default::default(),- });- let props = external_props(&enum_schema);- assert_eq!(- props.get("avro::doc"),- Some(&"enum documentation".to_string())- );- assert_eq!(- props.get("avro::aliases"),- Some(&"[fooenum,barenum]".to_string())- );- let fixed_schema = AvroSchema::Fixed(FixedSchema {- name: Name {- name: "fixed".to_string(),- namespace: None,- },- aliases: Some(vec![alias("foofixed"), alias("barfixed")]),- size: 1,- doc: None,- default: None,- attributes: Default::default(),- });- let props = external_props(&fixed_schema);- assert_eq!(- props.get("avro::aliases"),- Some(&"[foofixed,barfixed]".to_string())- );- }-- #[test]- fn test_invalid_avro_schema() {}-- #[test]- fn test_plain_types_schema() {- let schema = AvroSchema::parse_str(- r#"- {- "type" : "record",- "name" : "topLevelRecord",- "fields" : [ {- "name" : "id",- "type" : [ "int", "null" ]- }, {- "name" : "bool_col",- "type" : [ "boolean", "null" ]- }, {- "name" : "tinyint_col",- "type" : [ "int", "null" ]- }, {- "name" : "smallint_col",- "type" : [ "int", "null" ]- }, {- "name" : "int_col",- "type" : [ "int", "null" ]- }, {- "name" : "bigint_col",- "type" : [ "long", "null" ]- }, {- "name" : "float_col",- "type" : [ "float", "null" ]- }, {- "name" : "double_col",- "type" : [ "double", "null" ]- }, {- "name" : "date_string_col",- "type" : [ "bytes", "null" ]- }, {- "name" : "string_col",- "type" : [ "bytes", "null" ]- }, {- "name" : "timestamp_col",- "type" : [ {- "type" : "long",- "logicalType" : "timestamp-micros"- }, "null" ]- } ]- }"#,- );- assert!(schema.is_ok(), "{schema:?}");- let arrow_schema = to_arrow_schema(&schema.unwrap());- assert!(arrow_schema.is_ok(), "{arrow_schema:?}");- let expected = Schema::new(vec![- Field::new("id", Int32, true),- Field::new("bool_col", Boolean, true),- Field::new("tinyint_col", Int32, true),- Field::new("smallint_col", Int32, true),- Field::new("int_col", Int32, true),- Field::new("bigint_col", Int64, true),- Field::new("float_col", Float32, true),- Field::new("double_col", Float64, true),- Field::new("date_string_col", Binary, true),- Field::new("string_col", Binary, true),- Field::new("timestamp_col", Timestamp(Microsecond, None), true),- ]);- assert_eq!(arrow_schema.unwrap(), expected);- }-- #[test]- fn test_nested_schema() {- let avro_schema = apache_avro::Schema::parse_str(- r#"- {- "type": "record",- "name": "r1",- "fields": [- {- "name": "col1",- "type": [- "null",- {- "type": "record",- "name": "r2",- "fields": [- {- "name": "col2",- "type": "string"- },- {- "name": "col3",- "type": ["null", "string"],- "default": null- }- ]- }- ],- "default": null- }- ]- }"#,- )- .unwrap();- // should not use Avro Record names.- let expected_arrow_schema = Schema::new(vec![Field::new(- "col1",- arrow::datatypes::DataType::Struct(- vec![- Field::new("col2", Utf8, false),- Field::new("col3", Utf8, true),- ]- .into(),- ),- true,- )]);- assert_eq!(- to_arrow_schema(&avro_schema).unwrap(),- expected_arrow_schema- );- }-- #[test]- fn test_non_record_schema() {- let arrow_schema = to_arrow_schema(&AvroSchema::String);- assert!(arrow_schema.is_ok(), "{arrow_schema:?}");- assert_eq!(- arrow_schema.unwrap(),- Schema::new(vec![Field::new("", Utf8, false)])- );- }-}diff --git a/datafusion/datasource-avro/src/file_format.rs b/datafusion/datasource-avro/src/file_format.rsindex c4960dbcc9..35fd90524d 100644--- a/datafusion/datasource-avro/src/file_format.rs+++ b/datafusion/datasource-avro/src/file_format.rs@@ -16,13 +16,12 @@// under the License.//! Apache Avro [`FileFormat`] abstractions-use std::any::Any;use std::collections::HashMap;use std::fmt;use std::sync::Arc;-use crate::avro_to_arrow::read_avro_schema_from_reader;+use crate::read_avro_schema_from_reader;use crate::source::AvroSource;use arrow::datatypes::Schema;diff --git a/datafusion/datasource-avro/src/mod.rs b/datafusion/datasource-avro/src/mod.rsindex 5ad209591e..42cf2a3ab7 100644--- a/datafusion/datasource-avro/src/mod.rs+++ b/datafusion/datasource-avro/src/mod.rs@@ -27,9 +27,127 @@//! An [Avro](https://avro.apache.org/) based [`FileSource`](datafusion_datasource::file::FileSource) implementation and related functionality.-pub mod avro_to_arrow;pub mod file_format;pub mod source;-pub use apache_avro;+use arrow::datatypes::{DataType, Field, Fields, Schema, UnionFields};+pub use arrow_avro;+use arrow_avro::reader::ReaderBuilder;pub use file_format::*;+use std::io::{BufReader, Read};+use std::sync::Arc;++/// Read Avro schema given a reader+pub fn read_avro_schema_from_reader<R: Read>(+ reader: &mut R,+) -> datafusion_common::Result<Schema> {+ let avro_reader = ReaderBuilder::new().build(BufReader::new(reader))?;+ // Avro readers perform strict schema resolution rules (e.g. record identity checks)+ // that are stricter than DataFusion's table schema handling needs for inferred schemas.+ // Drop metadata from inferred schemas so runtime batches and inferred table schemas+ // compare consistently without requiring strict Avro metadata identity.+ Ok(strip_metadata_from_schema(avro_reader.schema().as_ref()))+}++fn strip_metadata_from_schema(schema: &Schema) -> Schema {+ let fields = schema+ .fields+ .into_iter()+ .map(|f| Arc::new(strip_metadata_from_field(f.as_ref())))+ .collect::<Fields>();+ // Intentionally drop schema-level metadata+ Schema::new(fields)+}++fn strip_metadata_from_field(field: &Field) -> Field {+ // Intentionally drop field-level metadata+ Field::new(+ field.name(),+ strip_metadata_from_data_type(field.data_type()),+ field.is_nullable(),+ )+}++fn strip_metadata_from_data_type(data_type: &DataType) -> DataType {+ match data_type {+ DataType::Struct(fields) => DataType::Struct(+ fields+ .iter()+ .map(|f| Arc::new(strip_metadata_from_field(f.as_ref())))+ .collect(),+ ),+ DataType::List(field) => {+ DataType::List(Arc::new(strip_metadata_from_field(field.as_ref())))+ }+ DataType::LargeList(field) => {+ DataType::LargeList(Arc::new(strip_metadata_from_field(field.as_ref())))+ }+ DataType::FixedSizeList(field, size) => DataType::FixedSizeList(+ Arc::new(strip_metadata_from_field(field.as_ref())),+ *size,+ ),+ DataType::Map(field, sorted) => {+ DataType::Map(Arc::new(strip_metadata_from_field(field.as_ref())), *sorted)+ }+ DataType::Union(fields, mode) => {+ let (type_ids, children): (Vec<_>, Vec<_>) = fields+ .iter()+ .map(|(type_id, field)| {+ (type_id, Arc::new(strip_metadata_from_field(field.as_ref())))+ })+ .unzip();++ DataType::Union(+ UnionFields::try_new(type_ids, children)+ .expect("existing union fields should remain valid"),+ *mode,+ )+ }+ _ => data_type.clone(),+ }+}++#[cfg(test)]+mod test {+ use super::*;+ use arrow::datatypes::{DataType, Field, TimeUnit};+ use datafusion_common::Result as DFResult;+ use datafusion_common::test_util::arrow_test_data;+ use std::fs::File;++ fn avro_test_file(name: &str) -> String {+ format!("{}/avro/{name}", arrow_test_data())+ }++ #[test]+ fn test_read_avro_schema_from_reader() -> DFResult<()> {+ let path = avro_test_file("alltypes_dictionary.avro");+ let mut file = File::open(&path)?;+ let file_schema = read_avro_schema_from_reader(&mut file)?;++ let expected_fields = vec![+ Field::new("id", DataType::Int32, true),+ Field::new("bool_col", DataType::Boolean, true),+ Field::new("tinyint_col", DataType::Int32, true),+ Field::new("smallint_col", DataType::Int32, true),+ Field::new("int_col", DataType::Int32, true),+ Field::new("bigint_col", DataType::Int64, true),+ Field::new("float_col", DataType::Float32, true),+ Field::new("double_col", DataType::Float64, true),+ Field::new("date_string_col", DataType::Binary, true),+ Field::new("string_col", DataType::Binary, true),+ Field::new(+ "timestamp_col",+ DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),+ true,+ ),+ ];++ assert_eq!(file_schema.fields.len(), expected_fields.len());+ for (i, field) in file_schema.fields.iter().enumerate() {+ assert_eq!(field.as_ref(), &expected_fields[i]);+ }++ Ok(())+ }+}diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rsindex 525d72e620..06026bedf3 100644--- a/datafusion/datasource-avro/src/source.rs+++ b/datafusion/datasource-avro/src/source.rs@@ -20,8 +20,8 @@use std::any::Any;use std::sync::Arc;-use crate::avro_to_arrow::Reader as AvroReader;-+use arrow::datatypes::{Schema, SchemaRef};+use arrow_avro::reader::{Reader, ReaderBuilder};use datafusion_common::error::Result;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_datasource::TableSchema;@@ -29,7 +29,7 @@ use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::file_stream::FileOpener;use datafusion_datasource::projection::{ProjectionOpener, SplitProjection};-use datafusion_physical_expr_common::sort_expr::LexOrdering;+use datafusion_physical_expr_adapter::BatchAdapterFactory;use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;use datafusion_physical_plan::projection::ProjectionExprs;@@ -56,21 +56,60 @@ impl AvroSource {}}- fn open<R: std::io::Read>(&self, reader: R) -> Result<AvroReader<'static, R>> {+ fn open<R: std::io::BufRead>(+ &self,+ reader: R,+ projection: Option<Vec<usize>>,+ ) -> Result<Reader<R>> {+ let mut builder = ReaderBuilder::new()+ .with_batch_size(self.batch_size.expect("Batch size must set before open"));+ if let Some(projection) = projection {+ builder = builder.with_projection(projection);+ }+ builder.build(reader).map_err(Into::into)+ }++ fn projected_file_schema(&self) -> SchemaRef {let file_schema = self.table_schema.file_schema();- let projection = Some(+ if self.projection.file_indices.is_empty() {+ return Arc::clone(file_schema);+ }++ Arc::new(Schema::new(self.projection.file_indices.iter()- .map(|&idx| file_schema.field(idx).name().clone())+ .map(|idx| file_schema.field(*idx).clone()).collect::<Vec<_>>(),- );- AvroReader::try_new(- reader,- &Arc::clone(self.table_schema.file_schema()),- self.batch_size.expect("Batch size must set before open"),- projection.as_ref(),- )+ ))+ }++ fn writer_projection_for_schema(+ &self,+ writer_schema: &Schema,+ target_schema: &Schema,+ ) -> Option<Vec<usize>> {+ // `arrow-avro` accepts projection ordinals against the file's writer schema,+ // while DataFusion plans projection against the logical table schema. Remap+ // projected column names to writer ordinals so reader-level pushdown still+ // preserves DataFusion's existing name-based projection semantics.+ let projection = target_schema+ .fields()+ .iter()+ .filter_map(|field| {+ writer_schema+ .column_with_name(field.name())+ .map(|(idx, _)| idx)+ })+ .collect::<Vec<_>>();++ let identity_projection = projection.len() == writer_schema.fields().len()+ && projection+ .iter()+ .enumerate()+ .all(|(idx, value)| idx == *value);++ (!identity_projection).then_some(projection)}}@@ -131,14 +170,9 @@ impl FileSource for AvroSource {"avro"}- fn repartitioned(- &self,- _target_partitions: usize,- _repartition_file_min_size: usize,- _output_ordering: Option<LexOrdering>,- _config: &FileScanConfig,- ) -> Result<Option<FileScanConfig>> {- Ok(None)+ fn supports_repartitioning(&self) -> bool {+ // Avro OCF does not support safe byte-range splitting in this reader path.+ false}fn apply_expressions(@@ -158,6 +192,8 @@ impl FileSource for AvroSource {mod private {use super::*;+ use std::io::BufReader;+ use std::io::Seek;use bytes::Buf;use datafusion_datasource::{PartitionedFile, file_stream::FileOpenFuture};@@ -171,24 +207,57 @@ mod private {impl FileOpener for AvroOpener {fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {- let config = Arc::clone(&self.config);let object_store = Arc::clone(&self.object_store);+ let config = Arc::clone(&self.config);+ let projected_file_schema = config.projected_file_schema();+Ok(Box::pin(async move {let r = object_store.get(&partitioned_file.object_meta.location).await?;match r.payload {- GetResultPayload::File(file, _) => {- let reader = config.open(file)?;+ GetResultPayload::File(mut file, _) => {+ // Probe the writer schema first so logical projected columns can be+ // translated to the writer-schema ordinals expected by `arrow-avro`.+ let probe_reader =+ config.open(BufReader::new(file.try_clone()?), None)?;+ let writer_projection = config.writer_projection_for_schema(+ probe_reader.schema().as_ref(),+ projected_file_schema.as_ref(),+ );+ file.rewind()?;+ let reader =+ config.open(BufReader::new(file), writer_projection)?;+ let batch_adapter =+ BatchAdapterFactory::new(Arc::clone(&projected_file_schema))+ .make_adapter(&reader.schema())?;Ok(futures::stream::iter(reader)- .map(|r| r.map_err(Into::into))+ .map(move |r| {+ r.map_err(Into::into)+ .and_then(|batch| batch_adapter.adapt_batch(&batch))+ }).boxed())}GetResultPayload::Stream(_) => {let bytes = r.bytes().await?;- let reader = config.open(bytes.reader())?;+ // As above, inspect the writer schema before constructing the real+ // reader so `with_projection` can use writer-schema ordinals.+ let probe_reader =+ config.open(BufReader::new(bytes.clone().reader()), None)?;+ let writer_projection = config.writer_projection_for_schema(+ probe_reader.schema().as_ref(),+ projected_file_schema.as_ref(),+ );+ let reader = config+ .open(BufReader::new(bytes.reader()), writer_projection)?;+ let batch_adapter =+ BatchAdapterFactory::new(Arc::clone(&projected_file_schema))+ .make_adapter(&reader.schema())?;Ok(futures::stream::iter(reader)- .map(|r| r.map_err(Into::into))+ .map(move |r| {+ r.map_err(Into::into)+ .and_then(|batch| batch_adapter.adapt_batch(&batch))+ }).boxed())}}@@ -196,3 +265,249 @@ mod private {}}}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::*;+ use arrow::array::{+ BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array, Int64Array,+ TimestampMicrosecondArray, TimestampMillisecondArray,+ };+ use arrow::datatypes::TimeUnit;+ use arrow::datatypes::{DataType, Field};+ use std::fs::File;+ use std::io::BufReader;++ fn build_reader(+ name: &'_ str,+ projection: Option<Vec<usize>>,+ ) -> Reader<BufReader<File>> {+ let testdata = datafusion_common::test_util::arrow_test_data();+ let filename = format!("{testdata}/avro/{name}");+ let mut builder = ReaderBuilder::new().with_batch_size(64);+ if let Some(proj) = projection {+ builder = builder.with_projection(proj);+ }+ builder+ .build(BufReader::new(File::open(filename).unwrap()))+ .unwrap()+ }++ fn get_col<'a, T: 'static>(+ batch: &'a RecordBatch,+ col: (usize, &Field),+ ) -> Option<&'a T> {+ batch.column(col.0).as_any().downcast_ref::<T>()+ }++ #[test]+ fn test_avro_basic() {+ let mut reader = build_reader("alltypes_dictionary.avro", None);+ let batch = reader.next().unwrap().unwrap();++ assert_eq!(11, batch.num_columns());+ assert_eq!(2, batch.num_rows());++ let schema = reader.schema();+ let batch_schema = batch.schema();+ assert_eq!(schema, batch_schema);++ let id = schema.column_with_name("id").unwrap();+ assert_eq!(0, id.0);+ assert_eq!(&DataType::Int32, id.1.data_type());+ let col = get_col::<Int32Array>(&batch, id).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1, col.value(1));+ let bool_col = schema.column_with_name("bool_col").unwrap();+ assert_eq!(1, bool_col.0);+ assert_eq!(&DataType::Boolean, bool_col.1.data_type());+ let col = get_col::<BooleanArray>(&batch, bool_col).unwrap();+ assert!(col.value(0));+ assert!(!col.value(1));+ let tinyint_col = schema.column_with_name("tinyint_col").unwrap();+ assert_eq!(2, tinyint_col.0);+ assert_eq!(&DataType::Int32, tinyint_col.1.data_type());+ let col = get_col::<Int32Array>(&batch, tinyint_col).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1, col.value(1));+ let smallint_col = schema.column_with_name("smallint_col").unwrap();+ assert_eq!(3, smallint_col.0);+ assert_eq!(&DataType::Int32, smallint_col.1.data_type());+ let col = get_col::<Int32Array>(&batch, smallint_col).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1, col.value(1));+ let int_col = schema.column_with_name("int_col").unwrap();+ assert_eq!(4, int_col.0);+ let col = get_col::<Int32Array>(&batch, int_col).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1, col.value(1));+ assert_eq!(&DataType::Int32, int_col.1.data_type());+ let col = get_col::<Int32Array>(&batch, int_col).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1, col.value(1));+ let bigint_col = schema.column_with_name("bigint_col").unwrap();+ assert_eq!(5, bigint_col.0);+ let col = get_col::<Int64Array>(&batch, bigint_col).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(10, col.value(1));+ assert_eq!(&DataType::Int64, bigint_col.1.data_type());+ let float_col = schema.column_with_name("float_col").unwrap();+ assert_eq!(6, float_col.0);+ let col = get_col::<Float32Array>(&batch, float_col).unwrap();+ assert_eq!(0.0, col.value(0));+ assert_eq!(1.1, col.value(1));+ assert_eq!(&DataType::Float32, float_col.1.data_type());+ let col = get_col::<Float32Array>(&batch, float_col).unwrap();+ assert_eq!(0.0, col.value(0));+ assert_eq!(1.1, col.value(1));+ let double_col = schema.column_with_name("double_col").unwrap();+ assert_eq!(7, double_col.0);+ assert_eq!(&DataType::Float64, double_col.1.data_type());+ let col = get_col::<Float64Array>(&batch, double_col).unwrap();+ assert_eq!(0.0, col.value(0));+ assert_eq!(10.1, col.value(1));+ let date_string_col = schema.column_with_name("date_string_col").unwrap();+ assert_eq!(8, date_string_col.0);+ assert_eq!(&DataType::Binary, date_string_col.1.data_type());+ let col = get_col::<BinaryArray>(&batch, date_string_col).unwrap();+ assert_eq!("01/01/09".as_bytes(), col.value(0));+ assert_eq!("01/01/09".as_bytes(), col.value(1));+ let string_col = schema.column_with_name("string_col").unwrap();+ assert_eq!(9, string_col.0);+ assert_eq!(&DataType::Binary, string_col.1.data_type());+ let col = get_col::<BinaryArray>(&batch, string_col).unwrap();+ assert_eq!("0".as_bytes(), col.value(0));+ assert_eq!("1".as_bytes(), col.value(1));+ let timestamp_col = schema.column_with_name("timestamp_col").unwrap();+ assert_eq!(10, timestamp_col.0);+ assert_eq!(+ &DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),+ timestamp_col.1.data_type()+ );+ let col = get_col::<TimestampMicrosecondArray>(&batch, timestamp_col).unwrap();+ assert_eq!(1230768000000000, col.value(0));+ assert_eq!(1230768060000000, col.value(1));+ }++ #[test]+ fn test_avro_with_projection() {+ // Test projection to filter and reorder columns+ let projection = vec![9, 7, 1]; // string_col, double_col, bool_col++ let mut reader = build_reader("alltypes_dictionary.avro", Some(projection));+ let batch = reader.next().unwrap().unwrap();++ // Only 3 columns should be present (not all 11)+ assert_eq!(3, batch.num_columns());+ assert_eq!(2, batch.num_rows());++ let schema = reader.schema();+ let batch_schema = batch.schema();+ assert_eq!(schema, batch_schema);++ // Verify columns are in the order specified in projection+ // First column should be string_col (was at index 9 in original)+ assert_eq!("string_col", schema.field(0).name());+ assert_eq!(&DataType::Binary, schema.field(0).data_type());+ let col = batch+ .column(0)+ .as_any()+ .downcast_ref::<BinaryArray>()+ .unwrap();+ assert_eq!("0".as_bytes(), col.value(0));+ assert_eq!("1".as_bytes(), col.value(1));++ // Second column should be double_col (was at index 7 in original)+ assert_eq!("double_col", schema.field(1).name());+ assert_eq!(&DataType::Float64, schema.field(1).data_type());+ let col = batch+ .column(1)+ .as_any()+ .downcast_ref::<Float64Array>()+ .unwrap();+ assert_eq!(0.0, col.value(0));+ assert_eq!(10.1, col.value(1));++ // Third column should be bool_col (was at index 1 in original)+ assert_eq!("bool_col", schema.field(2).name());+ assert_eq!(&DataType::Boolean, schema.field(2).data_type());+ let col = batch+ .column(2)+ .as_any()+ .downcast_ref::<BooleanArray>()+ .unwrap();+ assert!(col.value(0));+ assert!(!col.value(1));+ }++ #[test]+ fn test_avro_timestamp_logical_types() {+ let mut reader = build_reader("timestamp_logical_types.avro", None);+ let batch = reader.next().unwrap().unwrap();++ assert_eq!(7, batch.num_columns());+ assert_eq!(2, batch.num_rows());++ let schema = reader.schema();+ let ts_millis = schema.column_with_name("ts_millis").unwrap();+ assert_eq!(1, ts_millis.0);+ assert_eq!(+ &DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),+ ts_millis.1.data_type()+ );+ let col = get_col::<TimestampMillisecondArray>(&batch, ts_millis).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1_000, col.value(1));++ let ts_micros = schema.column_with_name("ts_micros").unwrap();+ assert_eq!(2, ts_micros.0);+ assert_eq!(+ &DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),+ ts_micros.1.data_type()+ );+ let col = get_col::<TimestampMicrosecondArray>(&batch, ts_micros).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1_000_000, col.value(1));++ let ts_nanos = schema.column_with_name("ts_nanos").unwrap();+ assert_eq!(3, ts_nanos.0);+ assert_eq!(+ &DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".into())),+ ts_nanos.1.data_type()+ );+ let col = get_col::<TimestampNanosecondArray>(&batch, ts_nanos).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1_000_000_000, col.value(1));++ let local_ts_millis = schema.column_with_name("local_ts_millis").unwrap();+ assert_eq!(4, local_ts_millis.0);+ assert_eq!(+ &DataType::Timestamp(TimeUnit::Millisecond, None),+ local_ts_millis.1.data_type()+ );+ let col = get_col::<TimestampMillisecondArray>(&batch, local_ts_millis).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1_000, col.value(1));++ let local_ts_micros = schema.column_with_name("local_ts_micros").unwrap();+ assert_eq!(5, local_ts_micros.0);+ assert_eq!(+ &DataType::Timestamp(TimeUnit::Microsecond, None),+ local_ts_micros.1.data_type()+ );+ let col = get_col::<TimestampMicrosecondArray>(&batch, local_ts_micros).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1_000_000, col.value(1));++ let local_ts_nanos = schema.column_with_name("local_ts_nanos").unwrap();+ assert_eq!(6, local_ts_nanos.0);+ assert_eq!(+ &DataType::Timestamp(TimeUnit::Nanosecond, None),+ local_ts_nanos.1.data_type()+ );+ let col = get_col::<TimestampNanosecondArray>(&batch, local_ts_nanos).unwrap();+ assert_eq!(0, col.value(0));+ assert_eq!(1_000_000_000, col.value(1));+ }+}diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex 3d17ed30d5..baa7dd8770 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -38,7 +38,7 @@ name = "datafusion_proto"default = ["parquet"]json = ["pbjson", "serde", "serde_json", "datafusion-proto-common/json"]parquet = ["datafusion-datasource-parquet", "datafusion-common/parquet", "datafusion/parquet"]-avro = ["datafusion-datasource-avro", "datafusion-common/avro"]+avro = ["datafusion-datasource-avro"]# Note to developers: do *not* add `datafusion` as a dependency in# this crate. See https://github.com/apache/datafusion/issues/17713diff --git a/datafusion/sqllogictest/test_files/avro.slt b/datafusion/sqllogictest/test_files/avro.sltindex 6e580006fe..a2ef9620e4 100644--- a/datafusion/sqllogictest/test_files/avro.slt+++ b/datafusion/sqllogictest/test_files/avro.slt@@ -105,6 +105,23 @@ CREATE EXTERNAL TABLE alltypes_plain_zstandard (STORED AS AVROLOCATION '../../testing/data/avro/alltypes_plain.zstandard.avro';+statement ok+CREATE EXTERNAL TABLE alltypes_plain_reordered (+ string_col VARCHAR,+ id INT,+ bool_col BOOLEAN NOT NULL,+ tinyint_col TINYINT NOT NULL,+ smallint_col SMALLINT NOT NULL,+ int_col INT NOT NULL,+ bigint_col BIGINT NOT NULL,+ float_col FLOAT NOT NULL,+ double_col DOUBLE NOT NULL,+ date_string_col BYTEA NOT NULL,+ timestamp_col TIMESTAMP NOT NULL,+)+STORED AS AVRO+LOCATION '../../testing/data/avro/alltypes_plain.avro';+statement okCREATE EXTERNAL TABLE single_nan (mycol FLOAT@@ -268,6 +285,21 @@ SELECT id, string_col, int_col, bigint_col FROM alltypes_plain ORDER BY id LIMIT3 1 1 104 0 0 0+# projection should follow column names even when external table column order differs+query TB+SELECT string_col, bool_col FROM alltypes_plain_reordered ORDER BY id LIMIT 2+----+0 true+1 false++# explain plan for reordered table projection path+query TT+EXPLAIN SELECT string_col, bool_col FROM alltypes_plain_reordered+----+logical_plan TableScan: alltypes_plain_reordered projection=[string_col, bool_col]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/avro/alltypes_plain.avro]]}, projection=[string_col, bool_col], file_type=avro++# Config resetstatement ok-reset datafusion.sql_parser.map_string_types_to_utf8view;\ No newline at end of file+reset datafusion.sql_parser.map_string_types_to_utf8view;diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.mdindex fcc7aed293..454a8de3d1 100644--- a/docs/source/library-user-guide/upgrading/54.0.0.md+++ b/docs/source/library-user-guide/upgrading/54.0.0.md@@ -77,7 +77,7 @@ fn apply_expressions(**Node whose only expressions are in `output_ordering()` (e.g. a synthetic test node with no owned expression fields):**-````rust,ignore+```rust,ignorefn apply_expressions(&self,f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,@@ -90,6 +90,7 @@ fn apply_expressions(}Ok(tnr)}+```### `ExecutionPlan::partition_statistics` now returns `Arc<Statistics>`@@ -101,7 +102,7 @@ fn apply_expressions(fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {Ok(Statistics::new_unknown(&self.schema()))}-````+```**After:**@@ -166,3 +167,36 @@ let is_async = (func.downcast_ref::<AsyncScalarUDF>().is_some();```++### Avro API and timestamp decoding changes++DataFusion has switched to use `arrow-avro` (see [#17861]) when reading avro files+which results in a few changes:++- `DataFusionError::AvroError` has been removed.+- `From<apache_avro::Error> for DataFusionError` has been removed.+- Avro crate re-export changed:+ - Before: `datafusion::apache_avro`+ - After: `datafusion::arrow_avro`+- **Avro timestamp logical type interpretation changed.** Notable effects:+ - Avro `timestamp-*` logical types are read as UTC timezone-aware Arrow+ timestamps (`Timestamp(..., Some("+00:00"))`)+ - Avro `local-timestamp-*` logical types remain timezone-naive+ (`Timestamp(..., None)`)++**Who is affected:**++- Users matching on `DataFusionError::AvroError`+- Users importing `datafusion::apache_avro`+- Users relying on previous Avro timestamp logical type behavior++**Migration guide:**++- Replace `datafusion::apache_avro` imports with `datafusion::arrow_avro`.+- Update error handling code that matches on `DataFusionError::AvroError` to use+ the current error surface.+- Validate timestamp handling where timezone semantics matter:+ `timestamp-*` is UTC timezone-aware, while `local-timestamp-*` is+ timezone-naive.++[#17861]: https://github.com/apache/datafusion/pull/17861diff --git a/testing b/testingindex 0d60ccae40..7df2b70baf 160000--- a/testing+++ b/testing@@ -1 +1 @@-Subproject commit 0d60ccae40d0e8f2d22c15fafb01c5d4be8c63a6+Subproject commit 7df2b70baf4f081ebf8e0c6bd22745cf3cbfd824