Apache DataFusion SQL Query Engine
Add write support to Avro format (#24925)
Pepijn Van Eeckhoudta7ea1201bccb
diff --git a/Cargo.lock b/Cargo.lockindex 18fde39a95..6d4c38d8d9 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1994,13 +1994,18 @@ dependencies = ["async-trait","bytes","datafusion-common",+ "datafusion-common-runtime","datafusion-datasource",+ "datafusion-execution",+ "datafusion-expr","datafusion-physical-expr-adapter",+ "datafusion-physical-expr-common","datafusion-physical-plan","datafusion-proto-models","datafusion-session","futures","object_store",+ "tokio",][[package]]diff --git a/datafusion/datasource-avro/Cargo.toml b/datafusion/datasource-avro/Cargo.tomlindex 70b675d63f..63e949e21d 100644--- a/datafusion/datasource-avro/Cargo.toml+++ b/datafusion/datasource-avro/Cargo.toml@@ -45,13 +45,18 @@ arrow-avro = { workspace = true }async-trait = { workspace = true }bytes = { workspace = true }datafusion-common = { workspace = true, features = ["object_store"] }+datafusion-common-runtime = { workspace = true }datafusion-datasource = { workspace = true }+datafusion-execution = { workspace = true }+datafusion-expr = { workspace = true }datafusion-physical-expr-adapter = { workspace = true }+datafusion-physical-expr-common = { workspace = true }datafusion-physical-plan = { workspace = true }datafusion-proto-models = { workspace = true, optional = true }datafusion-session = { workspace = true }futures = { workspace = true }object_store = { workspace = true }+tokio = { workspace = true }[dev-dependencies]diff --git a/datafusion/datasource-avro/src/file_format.rs b/datafusion/datasource-avro/src/file_format.rsindex 9d9d3279c0..93dad800c6 100644--- a/datafusion/datasource-avro/src/file_format.rs+++ b/datafusion/datasource-avro/src/file_format.rs@@ -17,29 +17,53 @@//! Apache Avro [`FileFormat`] abstractionsuse std::collections::HashMap;-use std::fmt;+use std::fmt::Debug;use std::sync::Arc;+use std::{fmt, io};use crate::read_avro_schema_from_reader;use crate::source::AvroSource;use arrow::datatypes::Schema;use arrow::datatypes::SchemaRef;-use datafusion_common::DEFAULT_AVRO_EXTENSION;+use arrow_avro::errors::AvroError;+use arrow_avro::writer::format::AvroOcfFormat;+use arrow_avro::writer::{AvroWriter, WriterBuilder};use datafusion_common::GetExt;use datafusion_common::internal_err;+use datafusion_common::not_impl_err;use datafusion_common::parsers::CompressionTypeVariant;-use datafusion_common::{Result, Statistics};+use datafusion_common::{DEFAULT_AVRO_EXTENSION, Diagnostic};+use datafusion_common::{DataFusionError, Result, Statistics, internal_datafusion_err};+use datafusion_common_runtime::{JoinSet, SpawnedTask};+use datafusion_datasource::display::FileGroupDisplay;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_format::{FileFormat, FileFormatFactory};use datafusion_datasource::file_scan_config::FileScanConfig;+use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};+use datafusion_datasource::sink::{DataSink, DataSinkExec};use datafusion_datasource::source::DataSourceExec;-use datafusion_physical_plan::ExecutionPlan;+use datafusion_datasource::write::demux::DemuxedStreamReceiver;+use datafusion_datasource::write::{+ ObjectWriterBuilder, SharedBuffer, get_writer_schema,+};+use datafusion_execution::{SendableRecordBatchStream, TaskContext};+use datafusion_expr::dml::InsertOp;+use datafusion_physical_expr_common::sort_expr::LexRequirement;+use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};use datafusion_session::Session;use async_trait::async_trait;use object_store::{GetResultPayload, ObjectMeta, ObjectStore, ObjectStoreExt};+use tokio::io::AsyncWriteExt;++/// Initial writing buffer size. Note this is just a size hint for efficiency. It+/// will grow beyond the set value if needed.+const INITIAL_BUFFER_BYTES: usize = 1048576;++/// If the buffered Avro data exceeds this size, it is flushed to object store+const BUFFER_FLUSH_BYTES: usize = 1024000;#[derive(Default)]/// Factory struct used to create [`AvroFormat`]@@ -66,7 +90,7 @@ impl FileFormatFactory for AvroFormatFactory {}}-impl fmt::Debug for AvroFormatFactory {+impl Debug for AvroFormatFactory {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {f.debug_struct("AvroFormatFactory").finish()}@@ -147,6 +171,22 @@ impl FileFormat for AvroFormat {Ok(DataSourceExec::from_data_source(conf))}+ async fn create_writer_physical_plan(+ &self,+ input: Arc<dyn ExecutionPlan>,+ _state: &dyn Session,+ conf: FileSinkConfig,+ order_requirements: Option<LexRequirement>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ if conf.insert_op != InsertOp::Append {+ return not_impl_err!("Overwrites are not implemented yet for Avro format");+ }++ let sink = Arc::new(AvroFileSink::new(conf));++ Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)+ }+fn file_source(&self,table_schema: datafusion_datasource::TableSchema,@@ -154,3 +194,150 @@ impl FileFormat for AvroFormat {Arc::new(AvroSource::new(table_schema))}}++/// Implements [`FileSink`] for Avro Object Container Files+struct AvroFileSink {+ config: FileSinkConfig,+}++impl AvroFileSink {+ fn new(config: FileSinkConfig) -> Self {+ Self { config }+ }+}++#[async_trait]+impl FileSink for AvroFileSink {+ fn config(&self) -> &FileSinkConfig {+ &self.config+ }++ async fn spawn_writer_tasks_and_join(+ &self,+ context: &Arc<TaskContext>,+ demux_task: SpawnedTask<Result<()>>,+ mut file_stream_rx: DemuxedStreamReceiver,+ object_store: Arc<dyn ObjectStore>,+ ) -> Result<u64> {+ let mut file_write_tasks: JoinSet<std::result::Result<usize, DataFusionError>> =+ JoinSet::new();++ let writer_schema = get_writer_schema(&self.config);+ while let Some((path, mut rx)) = file_stream_rx.recv().await {+ let shared_buffer = SharedBuffer::new(INITIAL_BUFFER_BYTES);+ let mut avro_writer: AvroWriter<SharedBuffer> =+ WriterBuilder::new(writer_schema.as_ref().clone())+ .build::<_, AvroOcfFormat>(shared_buffer.clone())+ .map_err(|e| {+ internal_datafusion_err!("Failed to create Avro writer: {e}")+ })?;+ let mut object_store_writer = ObjectWriterBuilder::new(+ FileCompressionType::UNCOMPRESSED,+ &path,+ Arc::clone(&object_store),+ )+ .with_buffer_size(Some(+ context+ .session_config()+ .options()+ .execution+ .objectstore_writer_buffer_size,+ ))+ .build()?;+ file_write_tasks.spawn(async move {+ let mut row_count = 0;+ while let Some(batch) = rx.recv().await {+ row_count += batch.num_rows();+ avro_writer+ .write(&batch)+ .map_err(|e| internal_datafusion_err!("{e}"))?;+ let mut buff_to_flush = shared_buffer.buffer.try_lock().unwrap();+ if buff_to_flush.len() > BUFFER_FLUSH_BYTES {+ object_store_writer+ .write_all(buff_to_flush.as_slice())+ .await?;+ buff_to_flush.clear();+ }+ }+ if let Err(e) = avro_writer.finish() {+ return Err(match e {+ AvroError::NYI(e) => DataFusionError::NotImplemented(e),+ AvroError::EOF(e) => DataFusionError::IoError(io::Error::new(+ io::ErrorKind::UnexpectedEof,+ e,+ )),+ AvroError::ArrowError(e) => DataFusionError::ArrowError(e, None),+ AvroError::External(e) => DataFusionError::External(e),+ AvroError::IoError(msg, e) => DataFusionError::IoError(e)+ .with_diagnostic(Diagnostic::new_error(msg, None)),+ _ => internal_datafusion_err!("{e}"),+ });+ }+ let final_buff = shared_buffer.buffer.try_lock().unwrap();++ object_store_writer.write_all(final_buff.as_slice()).await?;+ object_store_writer.shutdown().await?;+ Ok(row_count)+ });+ }++ let mut row_count = 0;+ while let Some(result) = file_write_tasks.join_next().await {+ match result {+ Ok(r) => {+ row_count += r?;+ }+ Err(e) => {+ if e.is_panic() {+ std::panic::resume_unwind(e.into_panic());+ } else {+ unreachable!();+ }+ }+ }+ }++ demux_task+ .join_unwind()+ .await+ .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;+ Ok(row_count as u64)+ }+}++impl Debug for AvroFileSink {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("AvroFileSink").finish()+ }+}++impl DisplayAs for AvroFileSink {+ fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match t {+ DisplayFormatType::Default | DisplayFormatType::Verbose => {+ write!(f, "AvroFileSink(file_groups=")?;+ FileGroupDisplay(&self.config.file_group).fmt_as(t, f)?;+ write!(f, ")")+ }+ DisplayFormatType::TreeRender => {+ writeln!(f, "format: avro")?;+ write!(f, "file={}", self.config.original_url)+ }+ }+ }+}++#[async_trait]+impl DataSink for AvroFileSink {+ fn schema(&self) -> &SchemaRef {+ self.config.output_schema()+ }++ async fn write_all(+ &self,+ data: SendableRecordBatchStream,+ context: &Arc<TaskContext>,+ ) -> Result<u64> {+ FileSink::write_all(self, data, context).await+ }+}diff --git a/datafusion/sqllogictest/test_files/copy_avro.slt b/datafusion/sqllogictest/test_files/copy_avro.sltnew file mode 100644index 0000000000..6fa23fd3e2--- /dev/null+++ b/datafusion/sqllogictest/test_files/copy_avro.slt@@ -0,0 +1,166 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# tests for `COPY ... TO ... STORED AS AVRO`++statement ok+create table source_table(col1 integer, col2 varchar) as values (1, 'Foo'), (2, 'Bar');++# Copy from table to a single Avro file+query I+COPY source_table TO 'test_files/scratch/copy_avro/table.avro' STORED AS AVRO;+----+2++# Validate single Avro file output+statement ok+CREATE EXTERNAL TABLE validate_avro_file+STORED AS AVRO+LOCATION 'test_files/scratch/copy_avro/table.avro';++query IT+select * from validate_avro_file;+----+1 Foo+2 Bar++# Copy from table to a directory of (multiple) Avro files+query I+COPY (select * from source_table UNION ALL select * from source_table) to 'test_files/scratch/copy_avro/table_dir' STORED AS AVRO;+----+4++# Validate multiple Avro file output+statement ok+CREATE EXTERNAL TABLE validate_avro_dir STORED AS AVRO LOCATION 'test_files/scratch/copy_avro/table_dir';++query IT+select * from validate_avro_dir order by col1, col2;+----+1 Foo+1 Foo+2 Bar+2 Bar++# Copy 0 rows to a single Avro file output+query I+COPY (SELECT 1 AS id WHERE FALSE) TO 'test_files/scratch/copy_avro/table_no_rows.avro' STORED AS AVRO;+----+0++statement ok+CREATE EXTERNAL TABLE validate_avro_no_rows STORED AS AVRO LOCATION 'test_files/scratch/copy_avro/table_no_rows.avro';++# validate the Avro file contains 0 rows.+query I+SELECT count(id) FROM validate_avro_no_rows;+----+0++# Copy to directory as partitioned files+query I+COPY (values (1, 'a', 'x'), (2, 'b', 'y'), (3, 'c', 'z')) TO 'test_files/scratch/copy_avro/partitioned_table/' STORED AS AVRO PARTITIONED BY (column2, column3);+----+3++# validate multiple partitioned Avro file output+statement ok+CREATE EXTERNAL TABLE validate_partitioned_avro STORED AS AVRO+LOCATION 'test_files/scratch/copy_avro/partitioned_table/' PARTITIONED BY (column2, column3);++query ITT+select * from validate_partitioned_avro order by column1, column2, column3;+----+1 a x+2 b y+3 c z++statement ok+CREATE EXTERNAL TABLE validate_partitioned_avro_a_x STORED AS AVRO+LOCATION 'test_files/scratch/copy_avro/partitioned_table/column2=a/column3=x';++query I+select * from validate_partitioned_avro_a_x order by column1;+----+1++# Copy a variety of data types, including nested/complex types (list, struct,+# map), to a single Avro file and validate the round trip. The schema is+# inferred from the self-describing Avro file rather than declared explicitly.+query I+COPY (values+ (arrow_cast(1, 'Int32'), arrow_cast(2, 'Int64'), arrow_cast(3.1, 'Float64'), 19968::date, true, 'x',+ make_array(1, 2, 3), named_struct('a', 1, 'b', 'foo'), MAP(['k1', 'k2'], [10, 20])),+ (arrow_cast(11, 'Int32'), arrow_cast(22, 'Int64'), arrow_cast(3.2, 'Float64'), 19969::date, false, 'y',+ make_array(4, 5), named_struct('a', 2, 'b', 'bar'), MAP(['k3'], [30]))+) TO 'test_files/scratch/copy_avro/table_types.avro' STORED AS AVRO;+----+2++statement ok+CREATE EXTERNAL TABLE validate_avro_types STORED AS AVRO LOCATION 'test_files/scratch/copy_avro/table_types.avro';++query IIRDBT???+select * from validate_avro_types order by column1;+----+1 2 3.1 2024-09-02 true x [1, 2, 3] {a: 1, b: foo} {k1: 10, k2: 20}+11 22 3.2 2024-09-03 false y [4, 5] {a: 2, b: bar} {k3: 30}++# EXPLAIN output for COPY ... STORED AS AVRO+query TT+EXPLAIN COPY source_table TO 'test_files/scratch/copy_avro/table_explain.avro' STORED AS AVRO;+----+logical_plan+01)CopyTo: format=avro output_url=test_files/scratch/copy_avro/table_explain.avro options: ()+02)--TableScan: source_table projection=[col1, col2]+physical_plan+01)DataSinkExec: sink=AvroFileSink(file_groups=[])+02)--DataSourceExec: partitions=1, partition_sizes=[1]++# EXPLAIN ANALYZE COPY ... STORED AS AVRO. Note the AvroFileSink does not+# currently report rows_written/bytes_written/elapsed_compute metrics.+query TT+EXPLAIN ANALYZE COPY (SELECT col1, upper(col2) AS col2_upper FROM source_table ORDER BY col1) TO 'test_files/scratch/copy_avro/table_metrics.avro' STORED AS AVRO;+----+Plan with Metrics+01)DataSinkExec: sink=AvroFileSink(file_groups=[]), metrics=[]+02)--SortExec: expr=[col1@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=<slt:ignore>, spill_count=0, spilled_bytes=0.0 B, spilled_rows=0]+03)----ProjectionExec: expr=[col1@0 as col1, upper(col2@1) as col2_upper], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, expr_0_eval_time=<slt:ignore>, expr_1_eval_time=<slt:ignore>]+04)------DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]++# Error case: Format not explicitly set and unable to infer extension from output path+query error DataFusion error: Invalid or Unsupported Configuration: Format not explicitly set and unable to get file extension! Use STORED AS to define file format.+EXPLAIN COPY source_table to 'test_files/scratch/copy_avro/no_format_specified';++# The following test writes sufficient data to exceed `datasource::file_format::avro::BUFFER_FLUSH_BYTES`+# so that we excercise the intermediate flush code path.+statement ok+CREATE EXTERNAL TABLE repeat_much STORED AS PARQUET LOCATION 'data/repeat_much.snappy.parquet';++query I+COPY repeat_much TO 'test_files/scratch/copy_avro/repeat_much.avro' STORED AS AVRO;+----+1957500++statement ok+CREATE EXTERNAL TABLE repeat_much_copy STORED AS AVRO LOCATION 'test_files/scratch/copy_avro/repeat_much.avro';++query I+SELECT sum(a) FROM repeat_much UNION ALL SELECT sum(a) FROM repeat_much_copy;+----+4798655628750+4798655628750\ No newline at end of file