Apache DataFusion SQL Query Engine
fix: respect DataFrameWriteOptions::with_single_file_output for paths without extensions (#19931)
Kumar Ujjawal36c0cda206db
diff --git a/datafusion-examples/examples/data_io/parquet_encrypted.rs b/datafusion-examples/examples/data_io/parquet_encrypted.rsindex d3cc6a121f..26361e9b52 100644--- a/datafusion-examples/examples/data_io/parquet_encrypted.rs+++ b/datafusion-examples/examples/data_io/parquet_encrypted.rs@@ -55,7 +55,7 @@ pub async fn parquet_encrypted() -> datafusion::common::Result<()> {// Create a temporary file location for the encrypted parquet filelet tmp_source = TempDir::new()?;- let tempfile = tmp_source.path().join("cars_encrypted");+ let tempfile = tmp_source.path().join("cars_encrypted.parquet");// Write encrypted parquetlet mut options = TableParquetOptions::default();diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 3845694407..a5de79b052 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -28,7 +28,7 @@ use datafusion_common::{use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};-use datafusion_datasource::file_sink_config::FileSinkConfig;+use datafusion_datasource::file_sink_config::{FileOutputMode, FileSinkConfig};#[expect(deprecated)]use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_datasource::{@@ -674,6 +674,7 @@ impl TableProvider for ListingTable {insert_op,keep_partition_by_columns,file_extension: self.options().format.get_ext(),+ file_output_mode: FileOutputMode::Automatic,};// For writes, we only use user-specified ordering (no file groups to derive from)diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 1e9f72501e..fadc6ad792 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -78,9 +78,11 @@ pub struct DataFrameWriteOptions {/// Controls how new data should be written to the table, determining whether/// to append, overwrite, or replace existing data.insert_op: InsertOp,- /// Controls if all partitions should be coalesced into a single output file- /// Generally will have slower performance when set to true.- single_file_output: bool,+ /// Controls if all partitions should be coalesced into a single output file.+ /// - `None`: Use automatic mode (extension-based heuristic)+ /// - `Some(true)`: Force single file output at exact path+ /// - `Some(false)`: Force directory output with generated filenames+ single_file_output: Option<bool>,/// Sets which columns should be used for hive-style partitioned writes by name./// Can be set to empty vec![] for non-partitioned writes.partition_by: Vec<String>,@@ -94,7 +96,7 @@ impl DataFrameWriteOptions {pub fn new() -> Self {DataFrameWriteOptions {insert_op: InsertOp::Append,- single_file_output: false,+ single_file_output: None,partition_by: vec![],sort_by: vec![],}@@ -108,9 +110,13 @@ impl DataFrameWriteOptions {/// Set the single_file_output value to true or false///- /// When set to true, an output file will always be created even if the DataFrame is empty+ /// - `true`: Force single file output at the exact path specified+ /// - `false`: Force directory output with generated filenames+ ///+ /// When not called, automatic mode is used (extension-based heuristic).+ /// When set to true, an output file will always be created even if the DataFrame is empty.pub fn with_single_file_output(mut self, single_file_output: bool) -> Self {- self.single_file_output = single_file_output;+ self.single_file_output = Some(single_file_output);self}@@ -125,6 +131,15 @@ impl DataFrameWriteOptions {self.sort_by = sort_by;self}++ /// Build the options HashMap to pass to CopyTo for sink configuration.+ fn build_sink_options(&self) -> HashMap<String, String> {+ let mut options = HashMap::new();+ if let Some(single_file) = self.single_file_output {+ options.insert("single_file_output".to_string(), single_file.to_string());+ }+ options+ }}impl Default for DataFrameWriteOptions {@@ -2040,6 +2055,8 @@ impl DataFrame {let file_type = format_as_file_type(format);+ let copy_options = options.build_sink_options();+let plan = if options.sort_by.is_empty() {self.plan} else {@@ -2052,7 +2069,7 @@ impl DataFrame {plan,path.into(),file_type,- HashMap::new(),+ copy_options,options.partition_by,)?.build()?;@@ -2108,6 +2125,8 @@ impl DataFrame {let file_type = format_as_file_type(format);+ let copy_options = options.build_sink_options();+let plan = if options.sort_by.is_empty() {self.plan} else {@@ -2120,7 +2139,7 @@ impl DataFrame {plan,path.into(),file_type,- Default::default(),+ copy_options,options.partition_by,)?.build()?;diff --git a/datafusion/core/src/dataframe/parquet.rs b/datafusion/core/src/dataframe/parquet.rsindex 6edf628e2d..54dadfd78c 100644--- a/datafusion/core/src/dataframe/parquet.rs+++ b/datafusion/core/src/dataframe/parquet.rs@@ -76,6 +76,8 @@ impl DataFrame {let file_type = format_as_file_type(format);+ let copy_options = options.build_sink_options();+let plan = if options.sort_by.is_empty() {self.plan} else {@@ -88,7 +90,7 @@ impl DataFrame {plan,path.into(),file_type,- Default::default(),+ copy_options,options.partition_by,)?.build()?;@@ -324,4 +326,156 @@ mod tests {Ok(())}++ /// Test FileOutputMode::SingleFile - explicitly request single file output+ /// for paths WITHOUT file extensions. This verifies the fix for the regression+ /// where extension heuristics ignored the explicit with_single_file_output(true).+ #[tokio::test]+ async fn test_file_output_mode_single_file() -> Result<()> {+ use arrow::array::Int32Array;+ use arrow::datatypes::{DataType, Field, Schema};+ use arrow::record_batch::RecordBatch;++ let ctx = SessionContext::new();+ let tmp_dir = TempDir::new()?;++ // Path WITHOUT .parquet extension - this is the key scenario+ let output_path = tmp_dir.path().join("data_no_ext");+ let output_path_str = output_path.to_str().unwrap();++ let df = ctx.read_batch(RecordBatch::try_new(+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])),+ vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],+ )?)?;++ // Explicitly request single file output+ df.write_parquet(+ output_path_str,+ DataFrameWriteOptions::new().with_single_file_output(true),+ None,+ )+ .await?;++ // Verify: output should be a FILE, not a directory+ assert!(+ output_path.is_file(),+ "Expected single file at {:?}, but got is_file={}, is_dir={}",+ output_path,+ output_path.is_file(),+ output_path.is_dir()+ );++ // Verify the file is readable as parquet+ let file = std::fs::File::open(&output_path)?;+ let reader = parquet::file::reader::SerializedFileReader::new(file)?;+ let metadata = reader.metadata();+ assert_eq!(metadata.num_row_groups(), 1);+ assert_eq!(metadata.file_metadata().num_rows(), 3);++ Ok(())+ }++ /// Test FileOutputMode::Automatic - uses extension heuristic.+ /// Path WITH extension -> single file; path WITHOUT extension -> directory.+ #[tokio::test]+ async fn test_file_output_mode_automatic() -> Result<()> {+ use arrow::array::Int32Array;+ use arrow::datatypes::{DataType, Field, Schema};+ use arrow::record_batch::RecordBatch;++ let ctx = SessionContext::new();+ let tmp_dir = TempDir::new()?;++ let schema =+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));+ let batch = RecordBatch::try_new(+ schema,+ vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],+ )?;++ // Case 1: Path WITH extension -> should create single file (Automatic mode)+ let output_with_ext = tmp_dir.path().join("data.parquet");+ let df = ctx.read_batch(batch.clone())?;+ df.write_parquet(+ output_with_ext.to_str().unwrap(),+ DataFrameWriteOptions::new(), // Automatic mode (default)+ None,+ )+ .await?;++ assert!(+ output_with_ext.is_file(),+ "Path with extension should be a single file, got is_file={}, is_dir={}",+ output_with_ext.is_file(),+ output_with_ext.is_dir()+ );++ // Case 2: Path WITHOUT extension -> should create directory (Automatic mode)+ let output_no_ext = tmp_dir.path().join("data_dir");+ let df = ctx.read_batch(batch)?;+ df.write_parquet(+ output_no_ext.to_str().unwrap(),+ DataFrameWriteOptions::new(), // Automatic mode (default)+ None,+ )+ .await?;++ assert!(+ output_no_ext.is_dir(),+ "Path without extension should be a directory, got is_file={}, is_dir={}",+ output_no_ext.is_file(),+ output_no_ext.is_dir()+ );++ Ok(())+ }++ /// Test FileOutputMode::Directory - explicitly request directory output+ /// even for paths WITH file extensions.+ #[tokio::test]+ async fn test_file_output_mode_directory() -> Result<()> {+ use arrow::array::Int32Array;+ use arrow::datatypes::{DataType, Field, Schema};+ use arrow::record_batch::RecordBatch;++ let ctx = SessionContext::new();+ let tmp_dir = TempDir::new()?;++ // Path WITH .parquet extension but explicitly requesting directory output+ let output_path = tmp_dir.path().join("output.parquet");+ let output_path_str = output_path.to_str().unwrap();++ let df = ctx.read_batch(RecordBatch::try_new(+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])),+ vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],+ )?)?;++ // Explicitly request directory output (single_file_output = false)+ df.write_parquet(+ output_path_str,+ DataFrameWriteOptions::new().with_single_file_output(false),+ None,+ )+ .await?;++ // Verify: output should be a DIRECTORY, not a single file+ assert!(+ output_path.is_dir(),+ "Expected directory at {:?}, but got is_file={}, is_dir={}",+ output_path,+ output_path.is_file(),+ output_path.is_dir()+ );++ // Verify the directory contains parquet file(s)+ let entries: Vec<_> = std::fs::read_dir(&output_path)?+ .filter_map(|e| e.ok())+ .collect();+ assert!(+ !entries.is_empty(),+ "Directory should contain at least one file"+ );++ Ok(())+ }}diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rsindex a6db617907..def3c0f35f 100644--- a/datafusion/core/src/datasource/file_format/parquet.rs+++ b/datafusion/core/src/datasource/file_format/parquet.rs@@ -130,7 +130,9 @@ mod tests {use datafusion_common::test_util::batches_to_string;use datafusion_common::{Result, ScalarValue};use datafusion_datasource::file_format::FileFormat;- use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};+ use datafusion_datasource::file_sink_config::{+ FileOutputMode, FileSink, FileSinkConfig,+ };use datafusion_datasource::{ListingTableUrl, PartitionedFile};use datafusion_datasource_parquet::{ParquetFormat, ParquetFormatFactory, ParquetSink,@@ -1547,6 +1549,7 @@ mod tests {insert_op: InsertOp::Overwrite,keep_partition_by_columns: false,file_extension: "parquet".into(),+ file_output_mode: FileOutputMode::Automatic,};let parquet_sink = Arc::new(ParquetSink::new(file_sink_config,@@ -1638,6 +1641,7 @@ mod tests {insert_op: InsertOp::Overwrite,keep_partition_by_columns: false,file_extension: "parquet".into(),+ file_output_mode: FileOutputMode::Automatic,};let parquet_sink = Arc::new(ParquetSink::new(file_sink_config,@@ -1728,6 +1732,7 @@ mod tests {insert_op: InsertOp::Overwrite,keep_partition_by_columns: false,file_extension: "parquet".into(),+ file_output_mode: FileOutputMode::Automatic,};let parquet_sink = Arc::new(ParquetSink::new(file_sink_config,diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex e7035910de..b1aa850284 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -23,7 +23,7 @@ use std::sync::Arc;use crate::datasource::file_format::file_type_to_format;use crate::datasource::listing::ListingTableUrl;-use crate::datasource::physical_plan::FileSinkConfig;+use crate::datasource::physical_plan::{FileOutputMode, FileSinkConfig};use crate::datasource::{DefaultTableSource, source_as_provider};use crate::error::{DataFusionError, Result};use crate::execution::context::{ExecutionProps, SessionState};@@ -549,8 +549,30 @@ impl DefaultPhysicalPlanner {}};+ // Parse single_file_output option if explicitly set+ let file_output_mode = match source_option_tuples+ .get("single_file_output")+ .map(|v| v.trim())+ {+ None => FileOutputMode::Automatic,+ Some("true") => FileOutputMode::SingleFile,+ Some("false") => FileOutputMode::Directory,+ Some(value) => {+ return Err(DataFusionError::Configuration(format!(+ "provided value for 'single_file_output' was not recognized: \"{value}\""+ )));+ }+ };++ // Filter out sink-related options that are not format options+ let format_options: HashMap<String, String> = source_option_tuples+ .iter()+ .filter(|(k, _)| k.as_str() != "single_file_output")+ .map(|(k, v)| (k.clone(), v.clone()))+ .collect();+let sink_format = file_type_to_format(file_type)?- .create(session_state, source_option_tuples)?;+ .create(session_state, &format_options)?;// Determine extension based on format extension and compressionlet file_extension = match sink_format.compression_type() {@@ -571,6 +593,7 @@ impl DefaultPhysicalPlanner {insert_op: InsertOp::Append,keep_partition_by_columns,file_extension,+ file_output_mode,};let ordering = input_exec.properties().output_ordering().cloned();diff --git a/datafusion/datasource/src/file_sink_config.rs b/datafusion/datasource/src/file_sink_config.rsindex 643831a119..1abce86a35 100644--- a/datafusion/datasource/src/file_sink_config.rs+++ b/datafusion/datasource/src/file_sink_config.rs@@ -32,6 +32,52 @@ use datafusion_expr::dml::InsertOp;use async_trait::async_trait;use object_store::ObjectStore;+/// Determines how `FileSink` output paths are interpreted.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]+pub enum FileOutputMode {+ /// Infer output mode from the output URL (for example, by extension / trailing `/`).+ #[default]+ Automatic,+ /// Write to a single output file at the exact output path.+ SingleFile,+ /// Write to a directory under the output path with generated filenames.+ Directory,+}++impl FileOutputMode {+ /// Resolve this mode into a `single_file_output` boolean for the demuxer.+ pub fn single_file_output(self, base_output_path: &ListingTableUrl) -> bool {+ match self {+ Self::Automatic => {+ !base_output_path.is_collection()+ && base_output_path.file_extension().is_some()+ }+ Self::SingleFile => true,+ Self::Directory => false,+ }+ }+}++impl From<Option<bool>> for FileOutputMode {+ fn from(value: Option<bool>) -> Self {+ match value {+ None => Self::Automatic,+ Some(true) => Self::SingleFile,+ Some(false) => Self::Directory,+ }+ }+}++impl From<FileOutputMode> for Option<bool> {+ fn from(value: FileOutputMode) -> Self {+ match value {+ FileOutputMode::Automatic => None,+ FileOutputMode::SingleFile => Some(true),+ FileOutputMode::Directory => Some(false),+ }+ }+}+/// General behaviors for files that do `DataSink` operations#[async_trait]pub trait FileSink: DataSink {@@ -112,6 +158,8 @@ pub struct FileSinkConfig {pub keep_partition_by_columns: bool,/// File extension without a dot(.)pub file_extension: String,+ /// Determines how the output path is interpreted.+ pub file_output_mode: FileOutputMode,}impl FileSinkConfig {diff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rsindex d7ac53d1a8..1648624747 100644--- a/datafusion/datasource/src/write/demux.rs+++ b/datafusion/datasource/src/write/demux.rs@@ -106,8 +106,9 @@ pub(crate) fn start_demuxer_task(let file_extension = config.file_extension.clone();let base_output_path = config.table_paths[0].clone();let task = if config.table_partition_cols.is_empty() {- let single_file_output = !base_output_path.is_collection()- && base_output_path.file_extension().is_some();+ let single_file_output = config+ .file_output_mode+ .single_file_output(&base_output_path);SpawnedTask::spawn(async move {row_count_demuxer(tx,diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 2efef9f12e..810ec6d1f1 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -759,6 +759,16 @@ message PartitionColumn {}+// Determines how file sink output paths are interpreted.+enum FileOutputMode {+ // Infer output mode from the URL (extension/trailing `/` heuristic).+ FILE_OUTPUT_MODE_AUTOMATIC = 0;+ // Write to a single file at the exact output path.+ FILE_OUTPUT_MODE_SINGLE_FILE = 1;+ // Write to a directory with generated filenames.+ FILE_OUTPUT_MODE_DIRECTORY = 2;+}+message FileSinkConfig {reserved 6; // writer_modereserved 8; // was `overwrite` which has been superseded by `insert_op`@@ -771,6 +781,8 @@ message FileSinkConfig {bool keep_partition_by_columns = 9;InsertOp insert_op = 10;string file_extension = 11;+ // Determines how the output path is interpreted.+ FileOutputMode file_output_mode = 12;}enum InsertOp {diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 00870d5ce1..7ed20785ab 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -5960,6 +5960,80 @@ impl<'de> serde::Deserialize<'de> for FileGroup {deserializer.deserialize_struct("datafusion.FileGroup", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for FileOutputMode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let variant = match self {+ Self::Automatic => "FILE_OUTPUT_MODE_AUTOMATIC",+ Self::SingleFile => "FILE_OUTPUT_MODE_SINGLE_FILE",+ Self::Directory => "FILE_OUTPUT_MODE_DIRECTORY",+ };+ serializer.serialize_str(variant)+ }+}+impl<'de> serde::Deserialize<'de> for FileOutputMode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "FILE_OUTPUT_MODE_AUTOMATIC",+ "FILE_OUTPUT_MODE_SINGLE_FILE",+ "FILE_OUTPUT_MODE_DIRECTORY",+ ];++ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = FileOutputMode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)+ })+ }++ fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)+ })+ }++ fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ match value {+ "FILE_OUTPUT_MODE_AUTOMATIC" => Ok(FileOutputMode::Automatic),+ "FILE_OUTPUT_MODE_SINGLE_FILE" => Ok(FileOutputMode::SingleFile),+ "FILE_OUTPUT_MODE_DIRECTORY" => Ok(FileOutputMode::Directory),+ _ => Err(serde::de::Error::unknown_variant(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_any(GeneratedVisitor)+ }+}impl serde::Serialize for FileRange {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -6382,6 +6456,9 @@ impl serde::Serialize for FileSinkConfig {if !self.file_extension.is_empty() {len += 1;}+ if self.file_output_mode != 0 {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.FileSinkConfig", len)?;if !self.object_store_url.is_empty() {struct_ser.serialize_field("objectStoreUrl", &self.object_store_url)?;@@ -6409,6 +6486,11 @@ impl serde::Serialize for FileSinkConfig {if !self.file_extension.is_empty() {struct_ser.serialize_field("fileExtension", &self.file_extension)?;}+ if self.file_output_mode != 0 {+ let v = FileOutputMode::try_from(self.file_output_mode)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.file_output_mode)))?;+ struct_ser.serialize_field("fileOutputMode", &v)?;+ }struct_ser.end()}}@@ -6435,6 +6517,8 @@ impl<'de> serde::Deserialize<'de> for FileSinkConfig {"insertOp","file_extension","fileExtension",+ "file_output_mode",+ "fileOutputMode",];#[allow(clippy::enum_variant_names)]@@ -6447,6 +6531,7 @@ impl<'de> serde::Deserialize<'de> for FileSinkConfig {KeepPartitionByColumns,InsertOp,FileExtension,+ FileOutputMode,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -6476,6 +6561,7 @@ impl<'de> serde::Deserialize<'de> for FileSinkConfig {"keepPartitionByColumns" | "keep_partition_by_columns" => Ok(GeneratedField::KeepPartitionByColumns),"insertOp" | "insert_op" => Ok(GeneratedField::InsertOp),"fileExtension" | "file_extension" => Ok(GeneratedField::FileExtension),+ "fileOutputMode" | "file_output_mode" => Ok(GeneratedField::FileOutputMode),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -6503,6 +6589,7 @@ impl<'de> serde::Deserialize<'de> for FileSinkConfig {let mut keep_partition_by_columns__ = None;let mut insert_op__ = None;let mut file_extension__ = None;+ let mut file_output_mode__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::ObjectStoreUrl => {@@ -6553,6 +6640,12 @@ impl<'de> serde::Deserialize<'de> for FileSinkConfig {}file_extension__ = Some(map_.next_value()?);}+ GeneratedField::FileOutputMode => {+ if file_output_mode__.is_some() {+ return Err(serde::de::Error::duplicate_field("fileOutputMode"));+ }+ file_output_mode__ = Some(map_.next_value::<FileOutputMode>()? as i32);+ }}}Ok(FileSinkConfig {@@ -6564,6 +6657,7 @@ impl<'de> serde::Deserialize<'de> for FileSinkConfig {keep_partition_by_columns: keep_partition_by_columns__.unwrap_or_default(),insert_op: insert_op__.unwrap_or_default(),file_extension: file_extension__.unwrap_or_default(),+ file_output_mode: file_output_mode__.unwrap_or_default(),})}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex 39d2604d45..0c9320c778 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -1186,6 +1186,9 @@ pub struct FileSinkConfig {pub insert_op: i32,#[prost(string, tag = "11")]pub file_extension: ::prost::alloc::string::String,+ /// Determines how the output path is interpreted.+ #[prost(enumeration = "FileOutputMode", tag = "12")]+ pub file_output_mode: i32,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct JsonSink {@@ -2254,6 +2257,39 @@ impl DateUnit {}}}+/// Determines how file sink output paths are interpreted.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]+pub enum FileOutputMode {+ /// Infer output mode from the URL (extension/trailing `/` heuristic).+ Automatic = 0,+ /// Write to a single file at the exact output path.+ SingleFile = 1,+ /// Write to a directory with generated filenames.+ Directory = 2,+}+impl FileOutputMode {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Automatic => "FILE_OUTPUT_MODE_AUTOMATIC",+ Self::SingleFile => "FILE_OUTPUT_MODE_SINGLE_FILE",+ Self::Directory => "FILE_OUTPUT_MODE_DIRECTORY",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "FILE_OUTPUT_MODE_AUTOMATIC" => Some(Self::Automatic),+ "FILE_OUTPUT_MODE_SINGLE_FILE" => Some(Self::SingleFile),+ "FILE_OUTPUT_MODE_DIRECTORY" => Some(Self::Directory),+ _ => None,+ }+ }+}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]pub enum InsertOp {diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex fc8eba12c5..e424be1626 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -821,6 +821,17 @@ impl TryFrom<&protobuf::FileSinkConfig> for FileSinkConfig {protobuf::InsertOp::Overwrite => InsertOp::Overwrite,protobuf::InsertOp::Replace => InsertOp::Replace,};+ let file_output_mode = match conf.file_output_mode() {+ protobuf::FileOutputMode::Automatic => {+ datafusion_datasource::file_sink_config::FileOutputMode::Automatic+ }+ protobuf::FileOutputMode::SingleFile => {+ datafusion_datasource::file_sink_config::FileOutputMode::SingleFile+ }+ protobuf::FileOutputMode::Directory => {+ datafusion_datasource::file_sink_config::FileOutputMode::Directory+ }+ };Ok(Self {original_url: String::default(),object_store_url: ObjectStoreUrl::parse(&conf.object_store_url)?,@@ -831,6 +842,7 @@ impl TryFrom<&protobuf::FileSinkConfig> for FileSinkConfig {insert_op,keep_partition_by_columns: conf.keep_partition_by_columns,file_extension: conf.file_extension.clone(),+ file_output_mode,})}}diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex f85b1d1e12..08ce00da4d 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -769,6 +769,17 @@ impl TryFrom<&FileSinkConfig> for protobuf::FileSinkConfig {})}).collect::<Result<Vec<_>>>()?;+ let file_output_mode = match conf.file_output_mode {+ datafusion_datasource::file_sink_config::FileOutputMode::Automatic => {+ protobuf::FileOutputMode::Automatic+ }+ datafusion_datasource::file_sink_config::FileOutputMode::SingleFile => {+ protobuf::FileOutputMode::SingleFile+ }+ datafusion_datasource::file_sink_config::FileOutputMode::Directory => {+ protobuf::FileOutputMode::Directory+ }+ };Ok(Self {object_store_url: conf.object_store_url.to_string(),file_groups,@@ -778,6 +789,7 @@ impl TryFrom<&FileSinkConfig> for protobuf::FileSinkConfig {keep_partition_by_columns: conf.keep_partition_by_columns,insert_op: conf.insert_op as i32,file_extension: conf.file_extension.to_string(),+ file_output_mode: file_output_mode.into(),})}}diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex f262020ab8..5bb771137f 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -36,7 +36,7 @@ use datafusion::datasource::listing::{};use datafusion::datasource::object_store::ObjectStoreUrl;use datafusion::datasource::physical_plan::{- FileGroup, FileScanConfigBuilder, FileSinkConfig, ParquetSource,+ FileGroup, FileOutputMode, FileScanConfigBuilder, FileSinkConfig, ParquetSource,wrap_partition_type_in_dict, wrap_partition_value_in_dict,};use datafusion::datasource::sink::DataSinkExec;@@ -1498,6 +1498,7 @@ fn roundtrip_json_sink() -> Result<()> {insert_op: InsertOp::Overwrite,keep_partition_by_columns: true,file_extension: "json".into(),+ file_output_mode: FileOutputMode::SingleFile,};let data_sink = Arc::new(JsonSink::new(file_sink_config,@@ -1536,6 +1537,7 @@ fn roundtrip_csv_sink() -> Result<()> {insert_op: InsertOp::Overwrite,keep_partition_by_columns: true,file_extension: "csv".into(),+ file_output_mode: FileOutputMode::Directory,};let data_sink = Arc::new(CsvSink::new(file_sink_config,@@ -1595,6 +1597,7 @@ fn roundtrip_parquet_sink() -> Result<()> {insert_op: InsertOp::Overwrite,keep_partition_by_columns: true,file_extension: "parquet".into(),+ file_output_mode: FileOutputMode::Automatic,};let data_sink = Arc::new(ParquetSink::new(file_sink_config,diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex 447a773eee..f5c4fc0921 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -21,7 +21,45 @@## DataFusion `53.0.0`-**Note:** DataFusion `53.0.0` has not been released yet. The information provided in this section pertains to features and changes that have already been merged to the main branch and are awaiting release in this version.+**Note:** DataFusion `53.0.0` has not been released yet. The information provided+*in this section pertains to features and changes that have already been merged+*to the main branch and are awaiting release in this version. See [#19692] for+\*more details.++[#19692]: https://github.com/apache/datafusion/issues/19692++### `FileSinkConfig` adds `file_output_mode`++`FileSinkConfig` now includes a `file_output_mode: FileOutputMode` field to control+single-file vs directory output behavior. Any code constructing `FileSinkConfig` via struct+literals must initialize this field.++The `FileOutputMode` enum has three variants:++- `Automatic` (default): Infer output mode from the URL (extension/trailing `/` heuristic)+- `SingleFile`: Write to a single file at the exact output path+- `Directory`: Write to a directory with generated filenames++**Before:**++```rust,ignore+FileSinkConfig {+ // ...+ file_extension: "parquet".into(),+}+```++**After:**++```rust,ignore+use datafusion_datasource::file_sink_config::FileOutputMode;++FileSinkConfig {+ // ...+ file_extension: "parquet".into(),+ file_output_mode: FileOutputMode::Automatic,+}+```### `SimplifyInfo` trait removed, `SimplifyContext` now uses builder-style API