Apache DataFusion SQL Query Engine
Add quote style and trimming to csv writier (#20813)
Xander85e75e2a19d2
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 85361ef5e1..abd58a556e 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -24,7 +24,7 @@ use crate::encryption::{FileDecryptionProperties, FileEncryptionProperties};use crate::error::_config_err;use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};use crate::parquet_config::DFParquetWriterVersion;-use crate::parsers::CompressionTypeVariant;+use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};use crate::utils::get_available_parallelism;use crate::{DataFusionError, Result};#[cfg(feature = "parquet_encryption")]@@ -2042,6 +2042,17 @@ impl ConfigField for CompressionTypeVariant {}}+impl ConfigField for CsvQuoteStyle {+ fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {+ v.some(key, self, description)+ }++ fn set(&mut self, _: &str, value: &str) -> Result<()> {+ *self = CsvQuoteStyle::from_str(value)?;+ Ok(())+ }+}+/// An implementation trait used to recursively walk configurationpub trait Visit {fn some<V: Display>(&mut self, key: &str, value: V, description: &'static str);@@ -3114,6 +3125,15 @@ config_namespace! {pub terminator: Option<u8>, default = Nonepub escape: Option<u8>, default = Nonepub double_quote: Option<bool>, default = None+ /// Quote style for CSV writing.+ /// One of: "Always", "Necessary", "NonNumeric", "Never"+ pub quote_style: CsvQuoteStyle, default = CsvQuoteStyle::Necessary+ /// Whether to ignore leading whitespace in string values when writing CSV.+ /// Defaults to `false` when `None`.+ pub ignore_leading_whitespace: Option<bool>, default = None+ /// Whether to ignore trailing whitespace in string values when writing CSV.+ /// Defaults to `false` when `None`.+ pub ignore_trailing_whitespace: Option<bool>, default = None/// Specifies whether newlines in (quoted) values are supported.////// Parsing newlines in quoted values may be affected by execution behaviour such as@@ -3222,6 +3242,30 @@ impl CsvOptions {self}+ /// Set the quote style for CSV writing.+ pub fn with_quote_style(mut self, quote_style: CsvQuoteStyle) -> Self {+ self.quote_style = quote_style;+ self+ }++ /// Set whether to ignore leading whitespace in string values when writing CSV.+ pub fn with_ignore_leading_whitespace(+ mut self,+ ignore_leading_whitespace: bool,+ ) -> Self {+ self.ignore_leading_whitespace = Some(ignore_leading_whitespace);+ self+ }++ /// Set whether to ignore trailing whitespace in string values when writing CSV.+ pub fn with_ignore_trailing_whitespace(+ mut self,+ ignore_trailing_whitespace: bool,+ ) -> Self {+ self.ignore_trailing_whitespace = Some(ignore_trailing_whitespace);+ self+ }+/// Specifies whether newlines in (quoted) values are supported.////// Parsing newlines in quoted values may be affected by execution behaviour such asdiff --git a/datafusion/common/src/file_options/csv_writer.rs b/datafusion/common/src/file_options/csv_writer.rsindex 4e6f74a444..fa116d1727 100644--- a/datafusion/common/src/file_options/csv_writer.rs+++ b/datafusion/common/src/file_options/csv_writer.rs@@ -94,6 +94,13 @@ impl TryFrom<&CsvOptions> for CsvWriterOptions {if let Some(v) = &value.double_quote {builder = builder.with_double_quote(*v)}+ builder = builder.with_quote_style(value.quote_style.into());+ if let Some(v) = &value.ignore_leading_whitespace {+ builder = builder.with_ignore_leading_whitespace(*v)+ }+ if let Some(v) = &value.ignore_trailing_whitespace {+ builder = builder.with_ignore_trailing_whitespace(*v)+ }Ok(CsvWriterOptions {writer_options: builder,compression: value.compression,diff --git a/datafusion/common/src/parsers.rs b/datafusion/common/src/parsers.rsindex cd3d607dac..6b930d110f 100644--- a/datafusion/common/src/parsers.rs+++ b/datafusion/common/src/parsers.rs@@ -73,3 +73,59 @@ impl CompressionTypeVariant {!matches!(self, &Self::UNCOMPRESSED)}}++/// CSV quote style+///+/// Controls when fields are quoted when writing CSV files.+/// Corresponds to [`arrow::csv::QuoteStyle`].+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]+pub enum CsvQuoteStyle {+ /// Quote all fields+ Always,+ /// Only quote fields when necessary (default)+ #[default]+ Necessary,+ /// Quote all non-numeric fields+ NonNumeric,+ /// Never quote fields+ Never,+}++impl FromStr for CsvQuoteStyle {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self, Self::Err> {+ match s.to_lowercase().as_str() {+ "always" => Ok(Self::Always),+ "necessary" => Ok(Self::Necessary),+ "non_numeric" | "nonnumeric" => Ok(Self::NonNumeric),+ "never" => Ok(Self::Never),+ _ => Err(DataFusionError::NotImplemented(format!(+ "Unsupported CSV quote style {s}"+ ))),+ }+ }+}++impl From<CsvQuoteStyle> for arrow::csv::QuoteStyle {+ fn from(style: CsvQuoteStyle) -> Self {+ match style {+ CsvQuoteStyle::Always => Self::Always,+ CsvQuoteStyle::NonNumeric => Self::NonNumeric,+ CsvQuoteStyle::Never => Self::Never,+ CsvQuoteStyle::Necessary => Self::Necessary,+ }+ }+}++impl Display for CsvQuoteStyle {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ let str = match self {+ Self::Always => "Always",+ Self::Necessary => "Necessary",+ Self::NonNumeric => "NonNumeric",+ Self::Never => "Never",+ };+ write!(f, "{str}")+ }+}diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex 53850e3146..87bcf3c144 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -434,6 +434,13 @@ message JsonWriterOptions {}+enum CsvQuoteStyle {+ NECESSARY = 0;+ ALWAYS = 1;+ NON_NUMERIC = 2;+ NEVER = 3;+}+message CsvWriterOptions {// Compression typeCompressionTypeVariant compression = 1;@@ -457,6 +464,12 @@ message CsvWriterOptions {string escape = 10;// Optional flag whether to double quotes, instead of escaping. Defaults to `true`bool double_quote = 11;+ // Quote style for CSV writing+ CsvQuoteStyle quote_style = 12;+ // Whether to ignore leading whitespace in string values+ bool ignore_leading_whitespace = 13;+ // Whether to ignore trailing whitespace in string values+ bool ignore_trailing_whitespace = 14;}// Options controlling CSV format@@ -480,6 +493,12 @@ message CsvOptions {bytes terminator = 17; // Optional terminator character as a bytebytes truncated_rows = 18; // Indicates if truncated rows are allowedoptional uint32 compression_level = 19; // Optional compression level+ // Quote style for CSV writing+ CsvQuoteStyle quote_style = 20;+ // Whether to ignore leading whitespace in string values+ bytes ignore_leading_whitespace = 21;+ // Whether to ignore trailing whitespace in string values+ bytes ignore_trailing_whitespace = 22;}// Options controlling CSV formatdiff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 5913a2c8f3..ef2a8c1847 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -22,7 +22,7 @@ use crate::common::proto_error;use crate::protobuf_common as protobuf;use arrow::array::{ArrayRef, AsArray};use arrow::buffer::Buffer;-use arrow::csv::WriterBuilder;+use arrow::csv::{QuoteStyle, WriterBuilder};use arrow::datatypes::{DataType, Field, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Schema,TimeUnit, UnionFields, UnionMode, i256,@@ -964,6 +964,17 @@ impl From<CompressionTypeVariant> for protobuf::CompressionTypeVariant {}}+impl From<protobuf::CsvQuoteStyle> for datafusion_common::parsers::CsvQuoteStyle {+ fn from(value: protobuf::CsvQuoteStyle) -> Self {+ match value {+ protobuf::CsvQuoteStyle::Necessary => Self::Necessary,+ protobuf::CsvQuoteStyle::Always => Self::Always,+ protobuf::CsvQuoteStyle::NonNumeric => Self::NonNumeric,+ protobuf::CsvQuoteStyle::Never => Self::Never,+ }+ }+}+impl TryFrom<&protobuf::CsvWriterOptions> for CsvWriterOptions {type Error = DataFusionError;@@ -1020,6 +1031,15 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions {.then(|| proto_opts.null_regex.clone()),comment: proto_opts.comment.first().copied(),truncated_rows: proto_opts.truncated_rows.first().map(|h| *h != 0),+ quote_style: proto_opts.quote_style().into(),+ ignore_leading_whitespace: proto_opts+ .ignore_leading_whitespace+ .first()+ .map(|h| *h != 0),+ ignore_trailing_whitespace: proto_opts+ .ignore_trailing_whitespace+ .first()+ .map(|h| *h != 0),})}}@@ -1281,6 +1301,16 @@ pub(crate) fn csv_writer_options_from_proto(return Err(proto_error("Error parsing CSV Escape"));}}+ let quote_style = match protobuf::CsvQuoteStyle::try_from(writer_options.quote_style)+ {+ Ok(protobuf::CsvQuoteStyle::Always) => QuoteStyle::Always,+ Ok(protobuf::CsvQuoteStyle::NonNumeric) => QuoteStyle::NonNumeric,+ Ok(protobuf::CsvQuoteStyle::Never) => QuoteStyle::Never,+ Ok(protobuf::CsvQuoteStyle::Necessary) => QuoteStyle::Necessary,+ _ => Err(proto_error(+ "Unknown quote style, must be one of: 'Always', 'NonNumeric', 'Never', 'Necessary'",+ ))?,+ };Ok(builder.with_header(writer_options.has_header).with_date_format(writer_options.date_format.clone())@@ -1288,7 +1318,10 @@ pub(crate) fn csv_writer_options_from_proto(.with_timestamp_format(writer_options.timestamp_format.clone()).with_time_format(writer_options.time_format.clone()).with_null(writer_options.null_value.clone())- .with_double_quote(writer_options.double_quote))+ .with_double_quote(writer_options.double_quote)+ .with_quote_style(quote_style)+ .with_ignore_leading_whitespace(writer_options.ignore_leading_whitespace)+ .with_ignore_trailing_whitespace(writer_options.ignore_trailing_whitespace))}#[cfg(test)]diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex 950b5a6f63..6112c55793 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -1867,6 +1867,15 @@ impl serde::Serialize for CsvOptions {if self.compression_level.is_some() {len += 1;}+ if self.quote_style != 0 {+ len += 1;+ }+ if !self.ignore_leading_whitespace.is_empty() {+ len += 1;+ }+ if !self.ignore_trailing_whitespace.is_empty() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion_common.CsvOptions", len)?;if !self.has_header.is_empty() {#[allow(clippy::needless_borrow)]@@ -1947,6 +1956,21 @@ impl serde::Serialize for CsvOptions {if let Some(v) = self.compression_level.as_ref() {struct_ser.serialize_field("compressionLevel", v)?;}+ if self.quote_style != 0 {+ let v = CsvQuoteStyle::try_from(self.quote_style)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.quote_style)))?;+ struct_ser.serialize_field("quoteStyle", &v)?;+ }+ if !self.ignore_leading_whitespace.is_empty() {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("ignoreLeadingWhitespace", pbjson::private::base64::encode(&self.ignore_leading_whitespace).as_str())?;+ }+ if !self.ignore_trailing_whitespace.is_empty() {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("ignoreTrailingWhitespace", pbjson::private::base64::encode(&self.ignore_trailing_whitespace).as_str())?;+ }struct_ser.end()}}@@ -1989,6 +2013,12 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {"truncatedRows","compression_level","compressionLevel",+ "quote_style",+ "quoteStyle",+ "ignore_leading_whitespace",+ "ignoreLeadingWhitespace",+ "ignore_trailing_whitespace",+ "ignoreTrailingWhitespace",];#[allow(clippy::enum_variant_names)]@@ -2012,6 +2042,9 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {Terminator,TruncatedRows,CompressionLevel,+ QuoteStyle,+ IgnoreLeadingWhitespace,+ IgnoreTrailingWhitespace,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -2052,6 +2085,9 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {"terminator" => Ok(GeneratedField::Terminator),"truncatedRows" | "truncated_rows" => Ok(GeneratedField::TruncatedRows),"compressionLevel" | "compression_level" => Ok(GeneratedField::CompressionLevel),+ "quoteStyle" | "quote_style" => Ok(GeneratedField::QuoteStyle),+ "ignoreLeadingWhitespace" | "ignore_leading_whitespace" => Ok(GeneratedField::IgnoreLeadingWhitespace),+ "ignoreTrailingWhitespace" | "ignore_trailing_whitespace" => Ok(GeneratedField::IgnoreTrailingWhitespace),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -2090,6 +2126,9 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {let mut terminator__ = None;let mut truncated_rows__ = None;let mut compression_level__ = None;+ let mut quote_style__ = None;+ let mut ignore_leading_whitespace__ = None;+ let mut ignore_trailing_whitespace__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::HasHeader => {@@ -2228,6 +2267,28 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0);}+ GeneratedField::QuoteStyle => {+ if quote_style__.is_some() {+ return Err(serde::de::Error::duplicate_field("quoteStyle"));+ }+ quote_style__ = Some(map_.next_value::<CsvQuoteStyle>()? as i32);+ }+ GeneratedField::IgnoreLeadingWhitespace => {+ if ignore_leading_whitespace__.is_some() {+ return Err(serde::de::Error::duplicate_field("ignoreLeadingWhitespace"));+ }+ ignore_leading_whitespace__ =+ Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::IgnoreTrailingWhitespace => {+ if ignore_trailing_whitespace__.is_some() {+ return Err(serde::de::Error::duplicate_field("ignoreTrailingWhitespace"));+ }+ ignore_trailing_whitespace__ =+ Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0)+ ;+ }}}Ok(CsvOptions {@@ -2250,12 +2311,92 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {terminator: terminator__.unwrap_or_default(),truncated_rows: truncated_rows__.unwrap_or_default(),compression_level: compression_level__,+ quote_style: quote_style__.unwrap_or_default(),+ ignore_leading_whitespace: ignore_leading_whitespace__.unwrap_or_default(),+ ignore_trailing_whitespace: ignore_trailing_whitespace__.unwrap_or_default(),})}}deserializer.deserialize_struct("datafusion_common.CsvOptions", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for CsvQuoteStyle {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let variant = match self {+ Self::Necessary => "NECESSARY",+ Self::Always => "ALWAYS",+ Self::NonNumeric => "NON_NUMERIC",+ Self::Never => "NEVER",+ };+ serializer.serialize_str(variant)+ }+}+impl<'de> serde::Deserialize<'de> for CsvQuoteStyle {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "NECESSARY",+ "ALWAYS",+ "NON_NUMERIC",+ "NEVER",+ ];++ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = CsvQuoteStyle;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)+ })+ }++ fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)+ })+ }++ fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ match value {+ "NECESSARY" => Ok(CsvQuoteStyle::Necessary),+ "ALWAYS" => Ok(CsvQuoteStyle::Always),+ "NON_NUMERIC" => Ok(CsvQuoteStyle::NonNumeric),+ "NEVER" => Ok(CsvQuoteStyle::Never),+ _ => Err(serde::de::Error::unknown_variant(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_any(GeneratedVisitor)+ }+}impl serde::Serialize for CsvWriterOptions {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -2297,6 +2438,15 @@ impl serde::Serialize for CsvWriterOptions {if self.double_quote {len += 1;}+ if self.quote_style != 0 {+ len += 1;+ }+ if self.ignore_leading_whitespace {+ len += 1;+ }+ if self.ignore_trailing_whitespace {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion_common.CsvWriterOptions", len)?;if self.compression != 0 {let v = CompressionTypeVariant::try_from(self.compression)@@ -2333,6 +2483,17 @@ impl serde::Serialize for CsvWriterOptions {if self.double_quote {struct_ser.serialize_field("doubleQuote", &self.double_quote)?;}+ if self.quote_style != 0 {+ let v = CsvQuoteStyle::try_from(self.quote_style)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.quote_style)))?;+ struct_ser.serialize_field("quoteStyle", &v)?;+ }+ if self.ignore_leading_whitespace {+ struct_ser.serialize_field("ignoreLeadingWhitespace", &self.ignore_leading_whitespace)?;+ }+ if self.ignore_trailing_whitespace {+ struct_ser.serialize_field("ignoreTrailingWhitespace", &self.ignore_trailing_whitespace)?;+ }struct_ser.end()}}@@ -2361,6 +2522,12 @@ impl<'de> serde::Deserialize<'de> for CsvWriterOptions {"escape","double_quote","doubleQuote",+ "quote_style",+ "quoteStyle",+ "ignore_leading_whitespace",+ "ignoreLeadingWhitespace",+ "ignore_trailing_whitespace",+ "ignoreTrailingWhitespace",];#[allow(clippy::enum_variant_names)]@@ -2376,6 +2543,9 @@ impl<'de> serde::Deserialize<'de> for CsvWriterOptions {Quote,Escape,DoubleQuote,+ QuoteStyle,+ IgnoreLeadingWhitespace,+ IgnoreTrailingWhitespace,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -2408,6 +2578,9 @@ impl<'de> serde::Deserialize<'de> for CsvWriterOptions {"quote" => Ok(GeneratedField::Quote),"escape" => Ok(GeneratedField::Escape),"doubleQuote" | "double_quote" => Ok(GeneratedField::DoubleQuote),+ "quoteStyle" | "quote_style" => Ok(GeneratedField::QuoteStyle),+ "ignoreLeadingWhitespace" | "ignore_leading_whitespace" => Ok(GeneratedField::IgnoreLeadingWhitespace),+ "ignoreTrailingWhitespace" | "ignore_trailing_whitespace" => Ok(GeneratedField::IgnoreTrailingWhitespace),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -2438,6 +2611,9 @@ impl<'de> serde::Deserialize<'de> for CsvWriterOptions {let mut quote__ = None;let mut escape__ = None;let mut double_quote__ = None;+ let mut quote_style__ = None;+ let mut ignore_leading_whitespace__ = None;+ let mut ignore_trailing_whitespace__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Compression => {@@ -2506,6 +2682,24 @@ impl<'de> serde::Deserialize<'de> for CsvWriterOptions {}double_quote__ = Some(map_.next_value()?);}+ GeneratedField::QuoteStyle => {+ if quote_style__.is_some() {+ return Err(serde::de::Error::duplicate_field("quoteStyle"));+ }+ quote_style__ = Some(map_.next_value::<CsvQuoteStyle>()? as i32);+ }+ GeneratedField::IgnoreLeadingWhitespace => {+ if ignore_leading_whitespace__.is_some() {+ return Err(serde::de::Error::duplicate_field("ignoreLeadingWhitespace"));+ }+ ignore_leading_whitespace__ = Some(map_.next_value()?);+ }+ GeneratedField::IgnoreTrailingWhitespace => {+ if ignore_trailing_whitespace__.is_some() {+ return Err(serde::de::Error::duplicate_field("ignoreTrailingWhitespace"));+ }+ ignore_trailing_whitespace__ = Some(map_.next_value()?);+ }}}Ok(CsvWriterOptions {@@ -2520,6 +2714,9 @@ impl<'de> serde::Deserialize<'de> for CsvWriterOptions {quote: quote__.unwrap_or_default(),escape: escape__.unwrap_or_default(),double_quote: double_quote__.unwrap_or_default(),+ quote_style: quote_style__.unwrap_or_default(),+ ignore_leading_whitespace: ignore_leading_whitespace__.unwrap_or_default(),+ ignore_trailing_whitespace: ignore_trailing_whitespace__.unwrap_or_default(),})}}diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex aac131e57e..4472ff0cde 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -619,6 +619,15 @@ pub struct CsvWriterOptions {/// Optional flag whether to double quotes, instead of escaping. Defaults to `true`#[prost(bool, tag = "11")]pub double_quote: bool,+ /// Quote style for CSV writing+ #[prost(enumeration = "CsvQuoteStyle", tag = "12")]+ pub quote_style: i32,+ /// Whether to ignore leading whitespace in string values+ #[prost(bool, tag = "13")]+ pub ignore_leading_whitespace: bool,+ /// Whether to ignore trailing whitespace in string values+ #[prost(bool, tag = "14")]+ pub ignore_trailing_whitespace: bool,}/// Options controlling CSV format#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]@@ -680,6 +689,15 @@ pub struct CsvOptions {/// Optional compression level#[prost(uint32, optional, tag = "19")]pub compression_level: ::core::option::Option<u32>,+ /// Quote style for CSV writing+ #[prost(enumeration = "CsvQuoteStyle", tag = "20")]+ pub quote_style: i32,+ /// Whether to ignore leading whitespace in string values+ #[prost(bytes = "vec", tag = "21")]+ pub ignore_leading_whitespace: ::prost::alloc::vec::Vec<u8>,+ /// Whether to ignore trailing whitespace in string values+ #[prost(bytes = "vec", tag = "22")]+ pub ignore_trailing_whitespace: ::prost::alloc::vec::Vec<u8>,}/// Options controlling CSV format#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]@@ -1207,6 +1225,38 @@ impl CompressionTypeVariant {}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]+pub enum CsvQuoteStyle {+ Necessary = 0,+ Always = 1,+ NonNumeric = 2,+ Never = 3,+}+impl CsvQuoteStyle {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Necessary => "NECESSARY",+ Self::Always => "ALWAYS",+ Self::NonNumeric => "NON_NUMERIC",+ Self::Never => "NEVER",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "NECESSARY" => Some(Self::Necessary),+ "ALWAYS" => Some(Self::Always),+ "NON_NUMERIC" => Some(Self::NonNumeric),+ "NEVER" => Some(Self::Never),+ _ => None,+ }+ }+}+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]pub enum JoinSide {LeftSide = 0,RightSide = 1,diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex a2da9b7799..dca95746b4 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -23,7 +23,7 @@ use crate::protobuf_common::{EmptyMessage, arrow_type::ArrowTypeEnum, scalar_value::Value,};use arrow::array::{ArrayRef, RecordBatch};-use arrow::csv::WriterBuilder;+use arrow::csv::{QuoteStyle, WriterBuilder};use arrow::datatypes::{DataType, Field, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Schema,SchemaRef, TimeUnit, UnionMode,@@ -31,6 +31,7 @@ use arrow::datatypes::{use arrow::ipc::writer::{CompressionContext, DictionaryTracker, IpcDataGenerator, IpcWriteOptions,};+use datafusion_common::parsers::CsvQuoteStyle;use datafusion_common::{Column, ColumnStatistics, Constraint, Constraints, DFSchema, DFSchemaRef,DataFusionError, JoinSide, ScalarValue, Statistics,@@ -852,6 +853,29 @@ impl From<&CompressionTypeVariant> for protobuf::CompressionTypeVariant {}}+impl From<CsvQuoteStyle> for protobuf::CsvQuoteStyle {+ fn from(value: CsvQuoteStyle) -> Self {+ match value {+ CsvQuoteStyle::Necessary => Self::Necessary,+ CsvQuoteStyle::Always => Self::Always,+ CsvQuoteStyle::NonNumeric => Self::NonNumeric,+ CsvQuoteStyle::Never => Self::Never,+ }+ }+}++impl From<QuoteStyle> for protobuf::CsvQuoteStyle {+ fn from(value: QuoteStyle) -> Self {+ match value {+ QuoteStyle::Necessary => Self::Necessary,+ QuoteStyle::Always => Self::Always,+ QuoteStyle::NonNumeric => Self::NonNumeric,+ QuoteStyle::Never => Self::Never,+ _ => Self::Necessary,+ }+ }+}+impl TryFrom<&CsvWriterOptions> for protobuf::CsvWriterOptions {type Error = DataFusionError;@@ -995,6 +1019,7 @@ impl TryFrom<&CsvOptions> for protobuf::CsvOptions {fn try_from(opts: &CsvOptions) -> datafusion_common::Result<Self, Self::Error> {let compression: protobuf::CompressionTypeVariant = opts.compression.into();+ let quote_style: protobuf::CsvQuoteStyle = opts.quote_style.into();Ok(protobuf::CsvOptions {has_header: opts.has_header.map_or_else(Vec::new, |h| vec![h as u8]),delimiter: vec![opts.delimiter],@@ -1017,6 +1042,13 @@ impl TryFrom<&CsvOptions> for protobuf::CsvOptions {comment: opts.comment.map_or_else(Vec::new, |h| vec![h]),truncated_rows: opts.truncated_rows.map_or_else(Vec::new, |h| vec![h as u8]),compression_level: opts.compression_level,+ quote_style: quote_style.into(),+ ignore_leading_whitespace: opts+ .ignore_leading_whitespace+ .map_or_else(Vec::new, |h| vec![h as u8]),+ ignore_trailing_whitespace: opts+ .ignore_trailing_whitespace+ .map_or_else(Vec::new, |h| vec![h as u8]),})}}@@ -1154,6 +1186,7 @@ pub(crate) fn csv_writer_options_to_proto(compression: &CompressionTypeVariant,) -> protobuf::CsvWriterOptions {let compression: protobuf::CompressionTypeVariant = compression.into();+ let quote_style: protobuf::CsvQuoteStyle = csv_options.quote_style().into();protobuf::CsvWriterOptions {compression: compression.into(),delimiter: (csv_options.delimiter() as char).to_string(),@@ -1166,5 +1199,8 @@ pub(crate) fn csv_writer_options_to_proto(quote: (csv_options.quote() as char).to_string(),escape: (csv_options.escape() as char).to_string(),double_quote: csv_options.double_quote(),+ quote_style: quote_style.into(),+ ignore_leading_whitespace: csv_options.ignore_leading_whitespace(),+ ignore_trailing_whitespace: csv_options.ignore_trailing_whitespace(),}}diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rsindex aac131e57e..4472ff0cde 100644--- a/datafusion/proto/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto/src/generated/datafusion_proto_common.rs@@ -619,6 +619,15 @@ pub struct CsvWriterOptions {/// Optional flag whether to double quotes, instead of escaping. Defaults to `true`#[prost(bool, tag = "11")]pub double_quote: bool,+ /// Quote style for CSV writing+ #[prost(enumeration = "CsvQuoteStyle", tag = "12")]+ pub quote_style: i32,+ /// Whether to ignore leading whitespace in string values+ #[prost(bool, tag = "13")]+ pub ignore_leading_whitespace: bool,+ /// Whether to ignore trailing whitespace in string values+ #[prost(bool, tag = "14")]+ pub ignore_trailing_whitespace: bool,}/// Options controlling CSV format#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]@@ -680,6 +689,15 @@ pub struct CsvOptions {/// Optional compression level#[prost(uint32, optional, tag = "19")]pub compression_level: ::core::option::Option<u32>,+ /// Quote style for CSV writing+ #[prost(enumeration = "CsvQuoteStyle", tag = "20")]+ pub quote_style: i32,+ /// Whether to ignore leading whitespace in string values+ #[prost(bytes = "vec", tag = "21")]+ pub ignore_leading_whitespace: ::prost::alloc::vec::Vec<u8>,+ /// Whether to ignore trailing whitespace in string values+ #[prost(bytes = "vec", tag = "22")]+ pub ignore_trailing_whitespace: ::prost::alloc::vec::Vec<u8>,}/// Options controlling CSV format#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]@@ -1207,6 +1225,38 @@ impl CompressionTypeVariant {}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]+pub enum CsvQuoteStyle {+ Necessary = 0,+ Always = 1,+ NonNumeric = 2,+ Never = 3,+}+impl CsvQuoteStyle {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Necessary => "NECESSARY",+ Self::Always => "ALWAYS",+ Self::NonNumeric => "NON_NUMERIC",+ Self::Never => "NEVER",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "NECESSARY" => Some(Self::Necessary),+ "ALWAYS" => Some(Self::Always),+ "NON_NUMERIC" => Some(Self::NonNumeric),+ "NEVER" => Some(Self::Never),+ _ => None,+ }+ }+}+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]pub enum JoinSide {LeftSide = 0,RightSide = 1,diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 4b111a49e4..a050f5fba2 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -17,11 +17,15 @@use std::sync::Arc;-use crate::protobuf::{CsvOptions as CsvOptionsProto, JsonOptions as JsonOptionsProto};+use super::LogicalExtensionCodec;+use crate::protobuf::{+ CsvOptions as CsvOptionsProto, CsvQuoteStyle as CsvQuoteStyleProto,+ JsonOptions as JsonOptionsProto,+};use datafusion_common::config::{CsvOptions, JsonOptions};use datafusion_common::{TableReference, exec_datafusion_err, exec_err, not_impl_err,- parsers::CompressionTypeVariant,+ parsers::{CompressionTypeVariant, CsvQuoteStyle},};use datafusion_datasource::file_format::FileFormatFactory;use datafusion_datasource_arrow::file_format::ArrowFormatFactory;@@ -30,8 +34,6 @@ use datafusion_datasource_json::file_format::JsonFormatFactory;use datafusion_execution::TaskContext;use prost::Message;-use super::LogicalExtensionCodec;-#[derive(Debug)]pub struct CsvLogicalExtensionCodec;@@ -63,6 +65,13 @@ impl CsvOptionsProto {.map_or(vec![], |v| vec![v as u8]),truncated_rows: options.truncated_rows.map_or(vec![], |v| vec![v as u8]),compression_level: options.compression_level,+ quote_style: options.quote_style as i32,+ ignore_leading_whitespace: options+ .ignore_leading_whitespace+ .map_or(vec![], |v| vec![v as u8]),+ ignore_trailing_whitespace: options+ .ignore_trailing_whitespace+ .map_or(vec![], |v| vec![v as u8]),}} else {CsvOptionsProto::default()@@ -154,6 +163,23 @@ impl From<&CsvOptionsProto> for CsvOptions {Some(proto.truncated_rows[0] != 0)},compression_level: proto.compression_level,+ quote_style: match CsvQuoteStyleProto::try_from(proto.quote_style) {+ Ok(CsvQuoteStyleProto::Always) => CsvQuoteStyle::Always,+ Ok(CsvQuoteStyleProto::NonNumeric) => CsvQuoteStyle::NonNumeric,+ Ok(CsvQuoteStyleProto::Never) => CsvQuoteStyle::Never,+ Ok(CsvQuoteStyleProto::Necessary) => CsvQuoteStyle::Necessary,+ _ => CsvQuoteStyle::Necessary,+ },+ ignore_leading_whitespace: if proto.ignore_leading_whitespace.is_empty() {+ None+ } else {+ Some(proto.ignore_leading_whitespace[0] != 0)+ },+ ignore_trailing_whitespace: if proto.ignore_trailing_whitespace.is_empty() {+ None+ } else {+ Some(proto.ignore_trailing_whitespace[0] != 0)+ },}}}diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.sltindex 5a7fa309db..d980e802c8 100644--- a/datafusion/sqllogictest/test_files/csv_files.slt+++ b/datafusion/sqllogictest/test_files/csv_files.slt@@ -380,3 +380,194 @@ SET datafusion.optimizer.repartition_file_min_size = 10485760;statement okdrop table stored_table_with_cr_terminator;++# Test quote_style option++statement ok+CREATE TABLE quote_style_source (+ int_col INT,+ string_col TEXT,+ float_col DOUBLE+) AS VALUES+(1, 'hello', 1.1),+(2, 'world', 2.2),+(3, 'comma,value', 3.3);++# QuoteStyle::Always - all fields are quoted+query I+COPY quote_style_source TO 'test_files/scratch/csv_files/quote_style_always.csv'+STORED AS csv+OPTIONS ('format.has_header' 'true', 'format.quote_style' 'Always');+----+3++# Read the file back using a delimiter that doesn't appear in the data,+# so each line is a single column and we can verify the actual quoting+statement ok+CREATE EXTERNAL TABLE stored_quote_style_always (+ whole_file TEXT+) STORED AS CSV+LOCATION 'test_files/scratch/csv_files/quote_style_always.csv'+OPTIONS ('format.has_header' 'true', 'format.delimiter' '@', 'format.quote' '~');++query T+select * from stored_quote_style_always;+----+"1","hello","1.1"+"2","world","2.2"+"3","comma,value","3.3"++statement ok+DROP TABLE stored_quote_style_always;++# QuoteStyle::NonNumeric - only string fields are quoted+query I+COPY quote_style_source TO 'test_files/scratch/csv_files/quote_style_nonnumeric.csv'+STORED AS csv+OPTIONS ('format.has_header' 'true', 'format.quote_style' 'NonNumeric');+----+3++# Read as a single column with a non-matching delimiter/quote to see raw output+statement ok+CREATE EXTERNAL TABLE stored_quote_style_nonnumeric (+ whole_file TEXT+) STORED AS CSV+LOCATION 'test_files/scratch/csv_files/quote_style_nonnumeric.csv'+OPTIONS ('format.has_header' 'true', 'format.delimiter' '@', 'format.quote' '~');++query T+select * from stored_quote_style_nonnumeric;+----+1,"hello",1.1+2,"world",2.2+3,"comma,value",3.3++statement ok+DROP TABLE stored_quote_style_nonnumeric;++# QuoteStyle::Never - no fields are quoted (can produce invalid CSV)+# Note: the comma in 'comma,value' will NOT be quoted, so reading back+# will see an extra column+query I+COPY quote_style_source TO 'test_files/scratch/csv_files/quote_style_never.csv'+STORED AS csv+OPTIONS ('format.has_header' 'true', 'format.quote_style' 'Never');+----+3++# Read as a single column with a non-matching delimiter/quote to see raw output+statement ok+CREATE EXTERNAL TABLE stored_quote_style_never (+ whole_file TEXT+) STORED AS CSV+LOCATION 'test_files/scratch/csv_files/quote_style_never.csv'+OPTIONS ('format.has_header' 'true', 'format.delimiter' '@', 'format.quote' '~');++# No fields are quoted - note the comma in 'comma,value' is unescaped+query T+select * from stored_quote_style_never;+----+1,hello,1.1+2,world,2.2+3,comma,value,3.3++statement ok+DROP TABLE stored_quote_style_never;++statement ok+DROP TABLE quote_style_source;++# Test ignore_leading_whitespace and ignore_trailing_whitespace options++statement ok+CREATE TABLE whitespace_source (+ id INT,+ value TEXT+) AS VALUES+(1, ' hello '),+(2, ' world '),+(3, 'no_space');++# Write with ignore_leading_whitespace to trim leading spaces+query I+COPY whitespace_source TO 'test_files/scratch/csv_files/trim_leading.csv'+STORED AS csv+OPTIONS ('format.has_header' 'true', 'format.ignore_leading_whitespace' 'true');+----+3++statement ok+CREATE EXTERNAL TABLE stored_trim_leading (+ id INT,+ value TEXT+) STORED AS CSV+LOCATION 'test_files/scratch/csv_files/trim_leading.csv'+OPTIONS ('format.has_header' 'true');++query ITT+select id, value, 'end' from stored_trim_leading order by id;+----+1 hello end+2 world end+3 no_space end++statement ok+DROP TABLE stored_trim_leading;++# Write with ignore_trailing_whitespace to trim trailing spaces+query I+COPY whitespace_source TO 'test_files/scratch/csv_files/trim_trailing.csv'+STORED AS csv+OPTIONS ('format.has_header' 'true', 'format.ignore_trailing_whitespace' 'true');+----+3++statement ok+CREATE EXTERNAL TABLE stored_trim_trailing (+ id INT,+ value TEXT+) STORED AS CSV+LOCATION 'test_files/scratch/csv_files/trim_trailing.csv'+OPTIONS ('format.has_header' 'true');++query ITT+select id, value, 'end' from stored_trim_trailing order by id;+----+1 hello end+2 world end+3 no_space end++statement ok+DROP TABLE stored_trim_trailing;++# Write with both ignore_leading and ignore_trailing whitespace+query I+COPY whitespace_source TO 'test_files/scratch/csv_files/trim_both.csv'+STORED AS csv+OPTIONS ('format.has_header' 'true',+ 'format.ignore_leading_whitespace' 'true',+ 'format.ignore_trailing_whitespace' 'true');+----+3++statement ok+CREATE EXTERNAL TABLE stored_trim_both (+ id INT,+ value TEXT+) STORED AS CSV+LOCATION 'test_files/scratch/csv_files/trim_both.csv'+OPTIONS ('format.has_header' 'true');++query ITT+select id, value, 'end' from stored_trim_both order by id;+----+1 hello end+2 world end+3 no_space end++statement ok+DROP TABLE stored_trim_both;++statement ok+DROP TABLE whitespace_source;