Apache DataFusion SQL Query Engine
Fix COPY TO does not produce an output file for the empty set (#18074)
Bert Vermeiren77a4cb741a68
diff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rsindex e800998230..52cb17c104 100644--- a/datafusion/datasource/src/write/demux.rs+++ b/datafusion/datasource/src/write/demux.rs@@ -40,9 +40,9 @@ use datafusion_common::cast::{};use datafusion_common::{exec_datafusion_err, internal_datafusion_err, not_impl_err};use datafusion_common_runtime::SpawnedTask;-use datafusion_execution::TaskContext;use chrono::NaiveDate;+use datafusion_execution::TaskContext;use futures::StreamExt;use object_store::path::Path;use rand::distr::SampleString;@@ -68,6 +68,11 @@ pub type DemuxedStreamReceiver = UnboundedReceiver<(Path, RecordBatchReceiver)>;/// be written with the extension from the path. Otherwise the default extension/// will be used and the output will be split into multiple files.///+/// Output file guarantees:+/// - Partitioned files: Files are created only for non-empty partitions.+/// - Single-file output: 1 file is always written, even when the stream is empty.+/// - Multi-file output: Depending on the number of record batches, 0 or more files are written.+////// Examples of `base_output_path`/// * `tmp/dataset/` -> is a folder since it ends in `/`/// * `tmp/dataset` -> is still a folder since it does not end in `/` but has no valid file extension@@ -171,6 +176,21 @@ async fn row_count_demuxer(max_rows_per_file};+ if single_file_output {+ // ensure we have one file open, even when the input stream is empty+ open_file_streams.push(create_new_file_stream(+ &base_output_path,+ &write_id,+ part_idx,+ &file_extension,+ single_file_output,+ max_buffered_batches,+ &mut tx,+ )?);+ row_counts.push(0);+ part_idx += 1;+ }+while let Some(rb) = input.next().await.transpose()? {// ensure we have at least minimum_parallel_files openif open_file_streams.len() < minimum_parallel_files {diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.sltindex 096cde86f2..9af0dc6393 100644--- a/datafusion/sqllogictest/test_files/copy.slt+++ b/datafusion/sqllogictest/test_files/copy.slt@@ -426,6 +426,21 @@ select * from validate_parquet_single;1 Foo2 Bar+# copy 0 rows to a single parquet file output+query I+COPY (SELECT 1 AS id WHERE FALSE) TO 'test_files/scratch/copy/table_no_rows.parquet';+----+0++statement ok+CREATE EXTERNAL TABLE validate_parquet_single_no_rows STORED AS PARQUET LOCATION 'test_files/scratch/copy/table_no_rows.parquet';++# validate the parquet file contains 0 rows.+query I+SELECT count(id) FROM validate_parquet_single_no_rows;+----+0+# copy from table to folder of compressed json filesquery ICOPY source_table to 'test_files/scratch/copy/table_json_gz' STORED AS JSON OPTIONS ('format.compression' gzip);