Skip to content

Apache DataFusion SQL Query Engine

Fix COPY TO does not produce an output file for the empty set (#18074)

Bert Vermeiren77a4cb741a68
diff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rs
index e800998230..52cb17c104 100644
--- a/datafusion/datasource/src/write/demux.rs
+++ b/datafusion/datasource/src/write/demux.rs
@@ -40,9 +40,9 @@ use datafusion_common::cast::{
};
use datafusion_common::{exec_datafusion_err, internal_datafusion_err, not_impl_err};
use datafusion_common_runtime::SpawnedTask;
-use datafusion_execution::TaskContext;
use chrono::NaiveDate;
+use datafusion_execution::TaskContext;
use futures::StreamExt;
use object_store::path::Path;
use rand::distr::SampleString;
@@ -68,6 +68,11 @@ pub type DemuxedStreamReceiver = UnboundedReceiver<(Path, RecordBatchReceiver)>;
/// be written with the extension from the path. Otherwise the default extension
/// will be used and the output will be split into multiple files.
///
+/// Output file guarantees:
+/// - Partitioned files: Files are created only for non-empty partitions.
+/// - Single-file output: 1 file is always written, even when the stream is empty.
+/// - Multi-file output: Depending on the number of record batches, 0 or more files are written.
+///
/// Examples of `base_output_path`
/// * `tmp/dataset/` -> is a folder since it ends in `/`
/// * `tmp/dataset` -> is still a folder since it does not end in `/` but has no valid file extension
@@ -171,6 +176,21 @@ async fn row_count_demuxer(
max_rows_per_file
};
+ if single_file_output {
+ // ensure we have one file open, even when the input stream is empty
+ open_file_streams.push(create_new_file_stream(
+ &base_output_path,
+ &write_id,
+ part_idx,
+ &file_extension,
+ single_file_output,
+ max_buffered_batches,
+ &mut tx,
+ )?);
+ row_counts.push(0);
+ part_idx += 1;
+ }
+
while let Some(rb) = input.next().await.transpose()? {
// ensure we have at least minimum_parallel_files open
if open_file_streams.len() < minimum_parallel_files {
diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.slt
index 096cde86f2..9af0dc6393 100644
--- a/datafusion/sqllogictest/test_files/copy.slt
+++ b/datafusion/sqllogictest/test_files/copy.slt
@@ -426,6 +426,21 @@ select * from validate_parquet_single;
1 Foo
2 Bar
+# copy 0 rows to a single parquet file output
+query I
+COPY (SELECT 1 AS id WHERE FALSE) TO 'test_files/scratch/copy/table_no_rows.parquet';
+----
+0
+
+statement ok
+CREATE EXTERNAL TABLE validate_parquet_single_no_rows STORED AS PARQUET LOCATION 'test_files/scratch/copy/table_no_rows.parquet';
+
+# validate the parquet file contains 0 rows.
+query I
+SELECT count(id) FROM validate_parquet_single_no_rows;
+----
+0
+
# copy from table to folder of compressed json files
query I
COPY source_table to 'test_files/scratch/copy/table_json_gz' STORED AS JSON OPTIONS ('format.compression' gzip);