Apache DataFusion SQL Query Engine
feat: add compression level configuration for JSON/CSV writers (#18954)
Smotrov Oleksii14cd71eaba76
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex c8ed491ef4..949fd83512 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -1725,6 +1725,7 @@ config_field!(bool, value => default_config_transform(value.to_lowercase().as_stconfig_field!(usize);config_field!(f64);config_field!(u64);+config_field!(u32);impl ConfigField for u8 {fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {@@ -2844,6 +2845,14 @@ config_namespace! {/// The default behaviour depends on the `datafusion.catalog.newlines_in_values` setting.pub newlines_in_values: Option<bool>, default = Nonepub compression: CompressionTypeVariant, default = CompressionTypeVariant::UNCOMPRESSED+ /// Compression level for the output file. The valid range depends on the+ /// compression algorithm:+ /// - ZSTD: 1 to 22 (default: 3)+ /// - GZIP: 0 to 9 (default: 6)+ /// - BZIP2: 0 to 9 (default: 6)+ /// - XZ: 0 to 9 (default: 6)+ /// If not specified, the default level for the compression algorithm is used.+ pub compression_level: Option<u32>, default = Nonepub schema_infer_max_rec: Option<usize>, default = Nonepub date_format: Option<String>, default = Nonepub datetime_format: Option<String>, default = None@@ -2966,6 +2975,14 @@ impl CsvOptions {self}+ /// Set the compression level for the output file.+ /// The valid range depends on the compression algorithm.+ /// If not specified, the default level for the algorithm is used.+ pub fn with_compression_level(mut self, level: u32) -> Self {+ self.compression_level = Some(level);+ self+ }+/// The delimiter character.pub fn delimiter(&self) -> u8 {self.delimiter@@ -2991,6 +3008,14 @@ config_namespace! {/// Options controlling JSON formatpub struct JsonOptions {pub compression: CompressionTypeVariant, default = CompressionTypeVariant::UNCOMPRESSED+ /// Compression level for the output file. The valid range depends on the+ /// compression algorithm:+ /// - ZSTD: 1 to 22 (default: 3)+ /// - GZIP: 0 to 9 (default: 6)+ /// - BZIP2: 0 to 9 (default: 6)+ /// - XZ: 0 to 9 (default: 6)+ /// If not specified, the default level for the compression algorithm is used.+ pub compression_level: Option<u32>, default = Nonepub schema_infer_max_rec: Option<usize>, default = None}}diff --git a/datafusion/common/src/file_options/csv_writer.rs b/datafusion/common/src/file_options/csv_writer.rsindex 943288af91..4e6f74a444 100644--- a/datafusion/common/src/file_options/csv_writer.rs+++ b/datafusion/common/src/file_options/csv_writer.rs@@ -31,6 +31,8 @@ pub struct CsvWriterOptions {/// Compression to apply after ArrowWriter serializes RecordBatches./// This compression is applied by DataFusion not the ArrowWriter itself.pub compression: CompressionTypeVariant,+ /// Compression level for the output file.+ pub compression_level: Option<u32>,}impl CsvWriterOptions {@@ -41,6 +43,20 @@ impl CsvWriterOptions {Self {writer_options,compression,+ compression_level: None,+ }+ }++ /// Create a new `CsvWriterOptions` with the specified compression level.+ pub fn new_with_level(+ writer_options: WriterBuilder,+ compression: CompressionTypeVariant,+ compression_level: u32,+ ) -> Self {+ Self {+ writer_options,+ compression,+ compression_level: Some(compression_level),}}}@@ -81,6 +97,7 @@ impl TryFrom<&CsvOptions> for CsvWriterOptions {Ok(CsvWriterOptions {writer_options: builder,compression: value.compression,+ compression_level: value.compression_level,})}}diff --git a/datafusion/common/src/file_options/json_writer.rs b/datafusion/common/src/file_options/json_writer.rsindex 750d297232..a537192c81 100644--- a/datafusion/common/src/file_options/json_writer.rs+++ b/datafusion/common/src/file_options/json_writer.rs@@ -27,11 +27,26 @@ use crate::{#[derive(Clone, Debug)]pub struct JsonWriterOptions {pub compression: CompressionTypeVariant,+ pub compression_level: Option<u32>,}impl JsonWriterOptions {pub fn new(compression: CompressionTypeVariant) -> Self {- Self { compression }+ Self {+ compression,+ compression_level: None,+ }+ }++ /// Create a new `JsonWriterOptions` with the specified compression and level.+ pub fn new_with_level(+ compression: CompressionTypeVariant,+ compression_level: u32,+ ) -> Self {+ Self {+ compression,+ compression_level: Some(compression_level),+ }}}@@ -41,6 +56,7 @@ impl TryFrom<&JsonOptions> for JsonWriterOptions {fn try_from(value: &JsonOptions) -> Result<Self> {Ok(JsonWriterOptions {compression: value.compression,+ compression_level: value.compression_level,})}}diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rsindex e81ae63011..7729f13901 100644--- a/datafusion/datasource-csv/src/file_format.rs+++ b/datafusion/datasource-csv/src/file_format.rs@@ -780,6 +780,7 @@ impl FileSink for CsvSink {context,serializer,self.writer_options.compression.into(),+ self.writer_options.compression_level,object_store,demux_task,file_stream_rx,diff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rsindex 0f8754ce72..a14458b5ac 100644--- a/datafusion/datasource-json/src/file_format.rs+++ b/datafusion/datasource-json/src/file_format.rs@@ -373,6 +373,7 @@ impl FileSink for JsonSink {context,serializer,self.writer_options.compression.into(),+ self.writer_options.compression_level,object_store,demux_task,file_stream_rx,diff --git a/datafusion/datasource/src/file_compression_type.rs b/datafusion/datasource/src/file_compression_type.rsindex ebc0e7821f..89efb58065 100644--- a/datafusion/datasource/src/file_compression_type.rs+++ b/datafusion/datasource/src/file_compression_type.rs@@ -155,22 +155,60 @@ impl FileCompressionType {}/// Wrap the given `BufWriter` so that it performs compressed writes- /// according to this `FileCompressionType`.+ /// according to this `FileCompressionType` using the default compression level.pub fn convert_async_writer(&self,w: BufWriter,) -> Result<Box<dyn AsyncWrite + Send + Unpin>> {+ self.convert_async_writer_with_level(w, None)+ }++ /// Wrap the given `BufWriter` so that it performs compressed writes+ /// according to this `FileCompressionType`.+ ///+ /// If `compression_level` is `Some`, the encoder will use the specified+ /// compression level. If `None`, the default level for each algorithm is used.+ pub fn convert_async_writer_with_level(+ &self,+ w: BufWriter,+ compression_level: Option<u32>,+ ) -> Result<Box<dyn AsyncWrite + Send + Unpin>> {+ #[cfg(feature = "compression")]+ use async_compression::Level;+Ok(match self.variant {#[cfg(feature = "compression")]- GZIP => Box::new(GzipEncoder::new(w)),+ GZIP => match compression_level {+ Some(level) => {+ Box::new(GzipEncoder::with_quality(w, Level::Precise(level as i32)))+ }+ None => Box::new(GzipEncoder::new(w)),+ },#[cfg(feature = "compression")]- BZIP2 => Box::new(BzEncoder::new(w)),+ BZIP2 => match compression_level {+ Some(level) => {+ Box::new(BzEncoder::with_quality(w, Level::Precise(level as i32)))+ }+ None => Box::new(BzEncoder::new(w)),+ },#[cfg(feature = "compression")]- XZ => Box::new(XzEncoder::new(w)),+ XZ => match compression_level {+ Some(level) => {+ Box::new(XzEncoder::with_quality(w, Level::Precise(level as i32)))+ }+ None => Box::new(XzEncoder::new(w)),+ },#[cfg(feature = "compression")]- ZSTD => Box::new(ZstdEncoder::new(w)),+ ZSTD => match compression_level {+ Some(level) => {+ Box::new(ZstdEncoder::with_quality(w, Level::Precise(level as i32)))+ }+ None => Box::new(ZstdEncoder::new(w)),+ },#[cfg(not(feature = "compression"))]GZIP | BZIP2 | XZ | ZSTD => {+ // compression_level is not used when compression feature is disabled+ let _ = compression_level;return Err(DataFusionError::NotImplemented("Compression feature is not enabled".to_owned(),));diff --git a/datafusion/datasource/src/write/mod.rs b/datafusion/datasource/src/write/mod.rsindex ca65477c49..e8d2d17da8 100644--- a/datafusion/datasource/src/write/mod.rs+++ b/datafusion/datasource/src/write/mod.rs@@ -131,6 +131,8 @@ pub struct ObjectWriterBuilder {object_store: Arc<dyn ObjectStore>,/// The size of the buffer for the object writer.buffer_size: Option<usize>,+ /// The compression level for the object writer.+ compression_level: Option<u32>,}impl ObjectWriterBuilder {@@ -145,6 +147,7 @@ impl ObjectWriterBuilder {location: location.clone(),object_store,buffer_size: None,+ compression_level: None,}}@@ -202,6 +205,22 @@ impl ObjectWriterBuilder {self.buffer_size}+ /// Set compression level for object writer.+ pub fn set_compression_level(&mut self, compression_level: Option<u32>) {+ self.compression_level = compression_level;+ }++ /// Set compression level for object writer, returning the builder.+ pub fn with_compression_level(mut self, compression_level: Option<u32>) -> Self {+ self.compression_level = compression_level;+ self+ }++ /// Currently specified compression level.+ pub fn get_compression_level(&self) -> Option<u32> {+ self.compression_level+ }+/// Return a writer object that writes to the object store location.////// If a buffer size has not been set, the default buffer buffer size will@@ -215,6 +234,7 @@ impl ObjectWriterBuilder {location,object_store,buffer_size,+ compression_level,} = self;let buf_writer = match buffer_size {@@ -222,6 +242,7 @@ impl ObjectWriterBuilder {None => BufWriter::new(object_store, location),};- file_compression_type.convert_async_writer(buf_writer)+ file_compression_type+ .convert_async_writer_with_level(buf_writer, compression_level)}}diff --git a/datafusion/datasource/src/write/orchestration.rs b/datafusion/datasource/src/write/orchestration.rsindex 1672817de0..39c91a1c0d 100644--- a/datafusion/datasource/src/write/orchestration.rs+++ b/datafusion/datasource/src/write/orchestration.rs@@ -248,6 +248,7 @@ pub async fn spawn_writer_tasks_and_join(context: &Arc<TaskContext>,serializer: Arc<dyn BatchSerializer>,compression: FileCompressionType,+ compression_level: Option<u32>,object_store: Arc<dyn ObjectStore>,demux_task: SpawnedTask<Result<()>>,mut file_stream_rx: DemuxedStreamReceiver,@@ -273,6 +274,7 @@ pub async fn spawn_writer_tasks_and_join(.execution.objectstore_writer_buffer_size,))+ .with_compression_level(compression_level).build()?;if tx_file_bundlediff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex 9d7d33dc01..08bb25bd71 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -461,12 +461,14 @@ message CsvOptions {bytes newlines_in_values = 16; // Indicates if newlines are supported in valuesbytes terminator = 17; // Optional terminator character as a bytebytes truncated_rows = 18; // Indicates if truncated rows are allowed+ optional uint32 compression_level = 19; // Optional compression level}// Options controlling CSV formatmessage JsonOptions {CompressionTypeVariant compression = 1; // Compression typeoptional uint64 schema_infer_max_rec = 2; // Optional max records for schema inference+ optional uint32 compression_level = 3; // Optional compression level}message TableParquetOptions {diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex c591a202af..7cb7a92031 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -908,6 +908,7 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions {double_quote: proto_opts.double_quote.first().map(|h| *h != 0),newlines_in_values: proto_opts.newlines_in_values.first().map(|h| *h != 0),compression: proto_opts.compression().into(),+ compression_level: proto_opts.compression_level,schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),date_format: (!proto_opts.date_format.is_empty()).then(|| proto_opts.date_format.clone()),@@ -1095,6 +1096,7 @@ impl TryFrom<&protobuf::JsonOptions> for JsonOptions {let compression: protobuf::CompressionTypeVariant = proto_opts.compression();Ok(JsonOptions {compression: compression.into(),+ compression_level: proto_opts.compression_level,schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),})}diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex 0bf87203ac..d38cf86825 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -1684,6 +1684,9 @@ impl serde::Serialize for CsvOptions {if !self.truncated_rows.is_empty() {len += 1;}+ if self.compression_level.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion_common.CsvOptions", len)?;if !self.has_header.is_empty() {#[allow(clippy::needless_borrow)]@@ -1761,6 +1764,9 @@ impl serde::Serialize for CsvOptions {#[allow(clippy::needless_borrows_for_generic_args)]struct_ser.serialize_field("truncatedRows", pbjson::private::base64::encode(&self.truncated_rows).as_str())?;}+ if let Some(v) = self.compression_level.as_ref() {+ struct_ser.serialize_field("compressionLevel", v)?;+ }struct_ser.end()}}@@ -1801,6 +1807,8 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {"terminator","truncated_rows","truncatedRows",+ "compression_level",+ "compressionLevel",];#[allow(clippy::enum_variant_names)]@@ -1823,6 +1831,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {NewlinesInValues,Terminator,TruncatedRows,+ CompressionLevel,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -1862,6 +1871,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {"newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues),"terminator" => Ok(GeneratedField::Terminator),"truncatedRows" | "truncated_rows" => Ok(GeneratedField::TruncatedRows),+ "compressionLevel" | "compression_level" => Ok(GeneratedField::CompressionLevel),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -1899,6 +1909,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {let mut newlines_in_values__ = None;let mut terminator__ = None;let mut truncated_rows__ = None;+ let mut compression_level__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::HasHeader => {@@ -2029,6 +2040,14 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0);}+ GeneratedField::CompressionLevel => {+ if compression_level__.is_some() {+ return Err(serde::de::Error::duplicate_field("compressionLevel"));+ }+ compression_level__ =+ map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0)+ ;+ }}}Ok(CsvOptions {@@ -2050,6 +2069,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {newlines_in_values: newlines_in_values__.unwrap_or_default(),terminator: terminator__.unwrap_or_default(),truncated_rows: truncated_rows__.unwrap_or_default(),+ compression_level: compression_level__,})}}@@ -4566,6 +4586,9 @@ impl serde::Serialize for JsonOptions {if self.schema_infer_max_rec.is_some() {len += 1;}+ if self.compression_level.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion_common.JsonOptions", len)?;if self.compression != 0 {let v = CompressionTypeVariant::try_from(self.compression)@@ -4577,6 +4600,9 @@ impl serde::Serialize for JsonOptions {#[allow(clippy::needless_borrows_for_generic_args)]struct_ser.serialize_field("schemaInferMaxRec", ToString::to_string(&v).as_str())?;}+ if let Some(v) = self.compression_level.as_ref() {+ struct_ser.serialize_field("compressionLevel", v)?;+ }struct_ser.end()}}@@ -4590,12 +4616,15 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {"compression","schema_infer_max_rec","schemaInferMaxRec",+ "compression_level",+ "compressionLevel",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Compression,SchemaInferMaxRec,+ CompressionLevel,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -4619,6 +4648,7 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {match value {"compression" => Ok(GeneratedField::Compression),"schemaInferMaxRec" | "schema_infer_max_rec" => Ok(GeneratedField::SchemaInferMaxRec),+ "compressionLevel" | "compression_level" => Ok(GeneratedField::CompressionLevel),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -4640,6 +4670,7 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {{let mut compression__ = None;let mut schema_infer_max_rec__ = None;+ let mut compression_level__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Compression => {@@ -4656,11 +4687,20 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0);}+ GeneratedField::CompressionLevel => {+ if compression_level__.is_some() {+ return Err(serde::de::Error::duplicate_field("compressionLevel"));+ }+ compression_level__ =+ map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0)+ ;+ }}}Ok(JsonOptions {compression: compression__.unwrap_or_default(),schema_infer_max_rec: schema_infer_max_rec__,+ compression_level: compression_level__,})}}diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex 9c4b7e1252..16601dcf46 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -649,6 +649,9 @@ pub struct CsvOptions {/// Indicates if truncated rows are allowed#[prost(bytes = "vec", tag = "18")]pub truncated_rows: ::prost::alloc::vec::Vec<u8>,+ /// Optional compression level+ #[prost(uint32, optional, tag = "19")]+ pub compression_level: ::core::option::Option<u32>,}/// Options controlling CSV format#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]@@ -659,6 +662,9 @@ pub struct JsonOptions {/// Optional max records for schema inference#[prost(uint64, optional, tag = "2")]pub schema_infer_max_rec: ::core::option::Option<u64>,+ /// Optional compression level+ #[prost(uint32, optional, tag = "3")]+ pub compression_level: ::core::option::Option<u32>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct TableParquetOptions {diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex ca1057da4e..dfa136717f 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -976,6 +976,7 @@ impl TryFrom<&CsvOptions> for protobuf::CsvOptions {null_regex: opts.null_regex.clone().unwrap_or_default(),comment: opts.comment.map_or_else(Vec::new, |h| vec![h]),truncated_rows: opts.truncated_rows.map_or_else(Vec::new, |h| vec![h as u8]),+ compression_level: opts.compression_level,})}}@@ -988,6 +989,7 @@ impl TryFrom<&JsonOptions> for protobuf::JsonOptions {Ok(protobuf::JsonOptions {compression: compression.into(),schema_infer_max_rec: opts.schema_infer_max_rec.map(|h| h as u64),+ compression_level: opts.compression_level,})}}diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rsindex 9c4b7e1252..16601dcf46 100644--- a/datafusion/proto/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto/src/generated/datafusion_proto_common.rs@@ -649,6 +649,9 @@ pub struct CsvOptions {/// Indicates if truncated rows are allowed#[prost(bytes = "vec", tag = "18")]pub truncated_rows: ::prost::alloc::vec::Vec<u8>,+ /// Optional compression level+ #[prost(uint32, optional, tag = "19")]+ pub compression_level: ::core::option::Option<u32>,}/// Options controlling CSV format#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]@@ -659,6 +662,9 @@ pub struct JsonOptions {/// Optional max records for schema inference#[prost(uint64, optional, tag = "2")]pub schema_infer_max_rec: ::core::option::Option<u64>,+ /// Optional compression level+ #[prost(uint32, optional, tag = "3")]+ pub compression_level: ::core::option::Option<u32>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct TableParquetOptions {diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 508edd3eff..87ce4d524f 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -62,6 +62,7 @@ impl CsvOptionsProto {.newlines_in_values.map_or(vec![], |v| vec![v as u8]),truncated_rows: options.truncated_rows.map_or(vec![], |v| vec![v as u8]),+ compression_level: options.compression_level,}} else {CsvOptionsProto::default()@@ -152,6 +153,7 @@ impl From<&CsvOptionsProto> for CsvOptions {} else {Some(proto.truncated_rows[0] != 0)},+ compression_level: proto.compression_level,}}}@@ -238,6 +240,7 @@ impl JsonOptionsProto {JsonOptionsProto {compression: options.compression as i32,schema_infer_max_rec: options.schema_infer_max_rec.map(|v| v as u64),+ compression_level: options.compression_level,}} else {JsonOptionsProto::default()@@ -256,6 +259,7 @@ impl From<&JsonOptionsProto> for JsonOptions {_ => CompressionTypeVariant::UNCOMPRESSED,},schema_infer_max_rec: proto.schema_infer_max_rec.map(|v| v as usize),+ compression_level: proto.compression_level,}}}diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex f1be8e207f..77676fc2fd 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -549,6 +549,8 @@ async fn roundtrip_logical_plan_copy_to_csv() -> Result<()> {csv_format.timestamp_format = Some("HH:mm:ss.SSSSSS".to_string());csv_format.time_format = Some("HH:mm:ss".to_string());csv_format.null_value = Some("NIL".to_string());+ csv_format.compression = CompressionTypeVariant::GZIP;+ csv_format.compression_level = Some(6);let file_type = format_as_file_type(Arc::new(CsvFormatFactory::new_with_options(csv_format.clone(),@@ -593,7 +595,9 @@ async fn roundtrip_logical_plan_copy_to_csv() -> Result<()> {assert_eq!(csv_format.datetime_format, csv_config.datetime_format);assert_eq!(csv_format.timestamp_format, csv_config.timestamp_format);assert_eq!(csv_format.time_format, csv_config.time_format);- assert_eq!(csv_format.null_value, csv_config.null_value)+ assert_eq!(csv_format.null_value, csv_config.null_value);+ assert_eq!(csv_format.compression, csv_config.compression);+ assert_eq!(csv_format.compression_level, csv_config.compression_level);}_ => panic!(),}