Apache DataFusion SQL Query Engine
fix: validate parquet.compression when it is set (#25626)
akashjainndb20a1ebe50f
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex e36fa80f53..02ab4f5b45 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -23,7 +23,9 @@ use arrow_ipc::CompressionType;use crate::encryption::{FileDecryptionProperties, FileEncryptionProperties};use crate::error::{_config_datafusion_err, _config_err};use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};-use crate::parquet_config::{DFParquetStatistics, DFParquetWriterVersion};+use crate::parquet_config::{+ DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion,+};use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};use crate::utils::get_available_parallelism;use crate::{DataFusionError, Result};@@ -1440,7 +1442,7 @@ config_namespace! {////// Note that this default setting is not the same as/// the default parquet writer setting.- pub compression: Option<String>, transform = str::to_lowercase, default = Some("zstd(3)".into())+ pub compression: Option<DFParquetCompression>, default = Some(DFParquetCompression::Zstd(3))/// (writing) Sets if dictionary encoding is enabled. If NULL, uses/// default parquet writer setting@@ -4684,6 +4686,85 @@ mod tests {assert_eq!(scalar, DFParquetStatistics::Page);}+ #[cfg(feature = "parquet")]+ #[test]+ fn test_parquet_compression_validation() {+ use crate::{config::ConfigOptions, parquet_config::DFParquetCompression};++ let mut config = ConfigOptions::default();+ assert_eq!(+ config.execution.parquet.compression,+ Some(DFParquetCompression::Zstd(3))+ );++ for (value, expected) in [+ ("snappy", DFParquetCompression::Snappy),+ ("GZIP(6)", DFParquetCompression::Gzip(6)),+ ("'zstd(22)'", DFParquetCompression::Zstd(22)),+ ] {+ config+ .set("datafusion.execution.parquet.compression", value)+ .unwrap();+ assert_eq!(config.execution.parquet.compression, Some(expected));+ }++ for (value, message) in [+ (+ "zstdd(3)",+ "Unknown or unsupported parquet compression: zstdd(3)",+ ),+ (+ "zstd",+ "zstd compression requires specifying a level such as zstd(4)",+ ),+ (+ "snappy(2)",+ "Compression snappy does not support specifying a level",+ ),+ ("zstd(23)", "Invalid compression level 23 for zstd"),+ ] {+ let err = config+ .set("datafusion.execution.parquet.compression", value)+ .unwrap_err();+ assert_contains!(err.to_string(), message);+ // A rejected value leaves the previous one in place.+ assert_eq!(+ config.execution.parquet.compression,+ Some(DFParquetCompression::Zstd(22))+ );+ }++ // An unset value can arise from deserialization. An invalid update must+ // leave that state unchanged rather than inserting the default.+ config.execution.parquet.compression = None;+ assert!(+ config+ .set("datafusion.execution.parquet.compression", "zstd")+ .is_err()+ );+ assert_eq!(config.execution.parquet.compression, None);++ config.execution.parquet.compression = Some(DFParquetCompression::Lz4);+ assert!(+ config+ .set("datafusion.execution.parquet.compression.typo", "snappy")+ .is_err()+ );+ assert!(+ config+ .reset("datafusion.execution.parquet.compression.typo")+ .is_err()+ );+ assert_eq!(+ config.execution.parquet.compression,+ Some(DFParquetCompression::Lz4)+ );++ let mut scalar = DFParquetCompression::Lz4;+ assert!(ConfigField::set(&mut scalar, "typo", "snappy").is_err());+ assert_eq!(scalar, DFParquetCompression::Lz4);+ }+#[cfg(feature = "parquet")]#[test]fn set_cdc_enabled_flag() {diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rsindex c50bb42a38..af4554c4ee 100644--- a/datafusion/common/src/file_options/parquet_writer.rs+++ b/datafusion/common/src/file_options/parquet_writer.rs@@ -22,13 +22,13 @@ use std::sync::Arc;use crate::{_internal_datafusion_err, DataFusionError, Result,config::{ParquetCdcOptions, ParquetOptions, TableParquetOptions},+ parquet_config::DFParquetCompression,};use arrow::datatypes::Schema;use parquet::arrow::encode_arrow_schema;use parquet::{arrow::ARROW_SCHEMA_META_KEY,- basic::{BrotliLevel, GzipLevel, ZstdLevel},file::{metadata::KeyValue,properties::{@@ -282,7 +282,7 @@ impl ParquetOptions {// We do not have access to default ColumnProperties set in Arrow.// Therefore, only overwrite if these settings exist.if let Some(compression) = compression {- builder = builder.set_compression(parse_compression_string(compression)?);+ builder = builder.set_compression((*compression).into());}if let Some(encoding) = encoding {builder = builder.set_encoding(parse_encoding_string(encoding)?);@@ -320,94 +320,11 @@ pub(crate) fn parse_encoding_string(}}-/// Splits compression string into compression codec and optional compression_level-/// I.e. gzip(2) -> gzip, 2-fn split_compression_string(str_setting: &str) -> Result<(String, Option<u32>)> {- // ignore string literal chars passed from sqlparser i.e. remove single quotes- let str_setting = str_setting.replace('\'', "");- let split_setting = str_setting.split_once('(');-- match split_setting {- Some((codec, rh)) => {- let level = &rh[..rh.len() - 1].parse::<u32>().map_err(|_| {- DataFusionError::Configuration(format!(- "Could not parse compression string. \- Got codec: {codec} and unknown level from {str_setting}"- ))- })?;- Ok((codec.to_owned(), Some(*level)))- }- None => Ok((str_setting.to_owned(), None)),- }-}--/// Helper to ensure compression codecs which don't support levels-/// don't have one set. E.g. snappy(2) is invalid.-fn check_level_is_none(codec: &str, level: Option<&u32>) -> Result<()> {- if level.is_some() {- return Err(DataFusionError::Configuration(format!(- "Compression {codec} does not support specifying a level"- )));- }- Ok(())-}--/// Helper to ensure compression codecs which require a level-/// do have one set. E.g. zstd is invalid, zstd(3) is valid-fn require_level(codec: &str, level: Option<u32>) -> Result<u32> {- level.ok_or(DataFusionError::Configuration(format!(- "{codec} compression requires specifying a level such as {codec}(4)"- )))-}-/// Parses datafusion.execution.parquet.compression String to a parquet::basic::Compressionpub fn parse_compression_string(str_setting: &str,) -> Result<parquet::basic::Compression> {- let str_setting_lower: &str = &str_setting.to_lowercase();- let (codec, level) = split_compression_string(str_setting_lower)?;- let codec = codec.as_str();- match codec {- "uncompressed" => {- check_level_is_none(codec, level.as_ref())?;- Ok(parquet::basic::Compression::UNCOMPRESSED)- }- "snappy" => {- check_level_is_none(codec, level.as_ref())?;- Ok(parquet::basic::Compression::SNAPPY)- }- "gzip" => {- let level = require_level(codec, level)?;- Ok(parquet::basic::Compression::GZIP(GzipLevel::try_new(- level,- )?))- }- "brotli" => {- let level = require_level(codec, level)?;- Ok(parquet::basic::Compression::BROTLI(BrotliLevel::try_new(- level,- )?))- }- "lz4" => {- check_level_is_none(codec, level.as_ref())?;- Ok(parquet::basic::Compression::LZ4)- }- "zstd" => {- let level = require_level(codec, level)?;- Ok(parquet::basic::Compression::ZSTD(ZstdLevel::try_new(- level as i32,- )?))- }- "lz4_raw" => {- check_level_is_none(codec, level.as_ref())?;- Ok(parquet::basic::Compression::LZ4_RAW)- }- _ => Err(DataFusionError::Configuration(format!(- "Unknown or unsupported parquet compression: \- {str_setting}. Valid values are: uncompressed, snappy, gzip(level), \- brotli(level), lz4, zstd(level), and lz4_raw."- ))),- }+ Ok(str_setting.parse::<DFParquetCompression>()?.into())}pub(crate) fn parse_statistics_string(str_setting: &str) -> Result<EnabledStatistics> {@@ -433,7 +350,9 @@ mod tests {MaxRowGroupBytes, ParquetCdcOptions, ParquetColumnOptions,ParquetEncryptionOptions, ParquetOptions,};- use crate::parquet_config::{DFParquetStatistics, DFParquetWriterVersion};+ use crate::parquet_config::{+ DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion,+ };use parquet::basic::Compression;use parquet::file::properties::{BloomFilterProperties, DEFAULT_BLOOM_FILTER_FPP, DEFAULT_BLOOM_FILTER_NDV,@@ -471,7 +390,7 @@ mod tests {data_pagesize_limit: 42,write_batch_size: 42,writer_version,- compression: Some("zstd(22)".into()),+ compression: Some(DFParquetCompression::Zstd(22)),dictionary_enabled: Some(!defaults.dictionary_enabled.unwrap_or(false)),dictionary_page_size_limit: 43,statistics_enabled: Some(DFParquetStatistics::Chunk),@@ -597,7 +516,12 @@ mod tests {// global options which set the default column propsencoding: default_col_props.encoding,- compression: default_col_props.compression,+ compression: match props.compression(&default_col) {+ Compression::ZSTD(level) => {+ Some(DFParquetCompression::Zstd(level.compression_level() as u32))+ }+ _ => None,+ },dictionary_enabled: default_col_props.dictionary_enabled,statistics_enabled: Some(props.statistics_enabled(&default_col).into()),bloom_filter_on_write: default_col_props@@ -770,7 +694,7 @@ mod tests {let mut from_extern_parquet =session_config_from_writer_props(&default_writer_props);from_extern_parquet.global.created_by = same_created_by;- from_extern_parquet.global.compression = Some("zstd(3)".into());+ from_extern_parquet.global.compression = Some(DFParquetCompression::Zstd(3));from_extern_parquet.global.skip_arrow_metadata = true;assert_eq!(diff --git a/datafusion/common/src/parquet_config.rs b/datafusion/common/src/parquet_config.rsindex b727664e77..c9710188e0 100644--- a/datafusion/common/src/parquet_config.rs+++ b/datafusion/common/src/parquet_config.rs@@ -221,3 +221,297 @@ impl From<parquet::file::properties::EnabledStatistics> for DFParquetStatistics}}}++/// Parquet compression codec, with a level for the codecs that take one+///+/// This enum validates compression settings at configuration time. An unknown+/// codec, a level on a codec that does not take one, a missing level on a codec+/// that requires one, or a level out of range is rejected by `SET` and by+/// deserialization instead of when the first file is written.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DFParquetCompression {+ /// No compression+ Uncompressed,+ /// Snappy+ Snappy,+ /// Gzip with a level from [`GZIP_LEVELS`]+ Gzip(u32),+ /// Brotli with a level from [`BROTLI_LEVELS`]+ Brotli(u32),+ /// LZ4 (Hadoop framed)+ Lz4,+ /// Zstd with a level from [`ZSTD_LEVELS`]+ Zstd(u32),+ /// LZ4 raw block format+ Lz4Raw,+}++/// Valid gzip levels, the same range as `parquet::basic::GzipLevel`+pub const GZIP_LEVELS: std::ops::RangeInclusive<u32> = 0..=9;+/// Valid brotli levels, the same range as `parquet::basic::BrotliLevel`+pub const BROTLI_LEVELS: std::ops::RangeInclusive<u32> = 0..=11;+/// Valid zstd levels. `parquet::basic::ZstdLevel` also accepts negative levels,+/// which the string form of this setting has never allowed.+pub const ZSTD_LEVELS: std::ops::RangeInclusive<u32> = 0..=22;++impl DFParquetCompression {+ /// Splits `gzip(2)` into `("gzip", Some(2))` and `snappy` into `("snappy", None)`+ fn split(setting: &str) -> Result<(&str, Option<u32>)> {+ let Some((codec, rest)) = setting.split_once('(') else {+ return Ok((setting, None));+ };+ let level = rest+ .strip_suffix(')')+ .and_then(|level| level.parse::<u32>().ok())+ .ok_or_else(|| {+ DataFusionError::Configuration(format!(+ "Could not parse compression string. \+ Got codec: {codec} and unknown level from {setting}"+ ))+ })?;+ Ok((codec, Some(level)))+ }++ fn without_level(codec: &str, level: Option<u32>) -> Result<()> {+ if level.is_some() {+ return Err(DataFusionError::Configuration(format!(+ "Compression {codec} does not support specifying a level"+ )));+ }+ Ok(())+ }++ fn level_in(+ codec: &str,+ level: Option<u32>,+ valid: std::ops::RangeInclusive<u32>,+ ) -> Result<u32> {+ let level = level.ok_or_else(|| {+ DataFusionError::Configuration(format!(+ "{codec} compression requires specifying a level such as {codec}(4)"+ ))+ })?;+ if !valid.contains(&level) {+ return Err(DataFusionError::Configuration(format!(+ "Invalid compression level {level} for {codec}. \+ Expected a level between {} and {}",+ valid.start(),+ valid.end()+ )));+ }+ Ok(level)+ }+}++impl FromStr for DFParquetCompression {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self, Self::Err> {+ // ignore string literal chars passed from sqlparser i.e. remove single quotes+ let setting = s.replace('\'', "").to_lowercase();+ let (codec, level) = Self::split(&setting)?;+ match codec {+ "uncompressed" => {+ Self::without_level(codec, level)?;+ Ok(Self::Uncompressed)+ }+ "snappy" => {+ Self::without_level(codec, level)?;+ Ok(Self::Snappy)+ }+ "gzip" => Ok(Self::Gzip(Self::level_in(codec, level, GZIP_LEVELS)?)),+ "brotli" => Ok(Self::Brotli(Self::level_in(codec, level, BROTLI_LEVELS)?)),+ "lz4" => {+ Self::without_level(codec, level)?;+ Ok(Self::Lz4)+ }+ "zstd" => Ok(Self::Zstd(Self::level_in(codec, level, ZSTD_LEVELS)?)),+ "lz4_raw" => {+ Self::without_level(codec, level)?;+ Ok(Self::Lz4Raw)+ }+ _ => Err(DataFusionError::Configuration(format!(+ "Unknown or unsupported parquet compression: \+ {s}. Valid values are: uncompressed, snappy, gzip(level), \+ brotli(level), lz4, zstd(level), and lz4_raw."+ ))),+ }+ }+}++impl Display for DFParquetCompression {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match self {+ Self::Uncompressed => f.write_str("uncompressed"),+ Self::Snappy => f.write_str("snappy"),+ Self::Gzip(level) => write!(f, "gzip({level})"),+ Self::Brotli(level) => write!(f, "brotli({level})"),+ Self::Lz4 => f.write_str("lz4"),+ Self::Zstd(level) => write!(f, "zstd({level})"),+ Self::Lz4Raw => f.write_str("lz4_raw"),+ }+ }+}++impl ConfigField for DFParquetCompression {+ fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {+ v.some(key, self, description)+ }++ fn set(&mut self, key: &str, value: &str) -> Result<()> {+ if !key.is_empty() {+ return crate::error::_config_err!(+ "Config field parquet.compression is a scalar DFParquetCompression and does not have nested field \"{}\"",+ key+ );+ }++ *self = Self::from_str(value)?;+ Ok(())+ }+}++/// `ConfigField` for `Option<DFParquetCompression>` parses before assigning so+/// an invalid value does not turn an unset option into the default.+impl ConfigField for Option<DFParquetCompression> {+ fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {+ match self {+ Some(compression) => compression.visit(v, key, description),+ None => v.none(key, description),+ }+ }++ fn set(&mut self, key: &str, value: &str) -> Result<()> {+ if !key.is_empty() {+ return crate::error::_config_err!(+ "Config field parquet.compression is a scalar Option<DFParquetCompression> and does not have nested field \"{}\"",+ key+ );+ }++ *self = Some(DFParquetCompression::from_str(value)?);+ Ok(())+ }++ fn reset(&mut self, key: &str) -> Result<()> {+ if key.is_empty() {+ *self = None;+ Ok(())+ } else {+ crate::error::_config_err!(+ "Config field parquet.compression is a scalar Option<DFParquetCompression> and does not have nested field \"{}\"",+ key+ )+ }+ }+}++#[cfg(feature = "parquet")]+impl From<DFParquetCompression> for parquet::basic::Compression {+ fn from(value: DFParquetCompression) -> Self {+ use parquet::basic::{BrotliLevel, GzipLevel, ZstdLevel};+ // The levels were checked against the same ranges when the value was parsed.+ match value {+ DFParquetCompression::Uncompressed => Self::UNCOMPRESSED,+ DFParquetCompression::Snappy => Self::SNAPPY,+ DFParquetCompression::Gzip(level) => {+ Self::GZIP(GzipLevel::try_new(level).expect("gzip level was validated"))+ }+ DFParquetCompression::Brotli(level) => Self::BROTLI(+ BrotliLevel::try_new(level).expect("brotli level was validated"),+ ),+ DFParquetCompression::Lz4 => Self::LZ4,+ DFParquetCompression::Zstd(level) => Self::ZSTD(+ ZstdLevel::try_new(level as i32).expect("zstd level was validated"),+ ),+ DFParquetCompression::Lz4Raw => Self::LZ4_RAW,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn parquet_compression_parses_and_displays() {+ for (input, expected, displayed) in [+ (+ "uncompressed",+ DFParquetCompression::Uncompressed,+ "uncompressed",+ ),+ ("SNAPPY", DFParquetCompression::Snappy, "snappy"),+ ("gzip(6)", DFParquetCompression::Gzip(6), "gzip(6)"),+ ("Brotli(11)", DFParquetCompression::Brotli(11), "brotli(11)"),+ ("lz4", DFParquetCompression::Lz4, "lz4"),+ ("'zstd(3)'", DFParquetCompression::Zstd(3), "zstd(3)"),+ ("lz4_raw", DFParquetCompression::Lz4Raw, "lz4_raw"),+ ("zstd(0)", DFParquetCompression::Zstd(0), "zstd(0)"),+ ] {+ let parsed: DFParquetCompression = input.parse().unwrap();+ assert_eq!(parsed, expected, "input: {input}");+ assert_eq!(parsed.to_string(), displayed, "input: {input}");+ assert_eq!(displayed.parse::<DFParquetCompression>().unwrap(), parsed);+ }+ }++ #[test]+ fn parquet_compression_rejects_invalid_settings() {+ for (input, message) in [+ (+ "zstdd(3)",+ "Unknown or unsupported parquet compression: zstdd(3)",+ ),+ (+ "zstd",+ "zstd compression requires specifying a level such as zstd(4)",+ ),+ (+ "snappy(2)",+ "Compression snappy does not support specifying a level",+ ),+ ("zstd(x)", "Could not parse compression string"),+ ("zstd(3", "Could not parse compression string"),+ (+ "zstd(23)",+ "Invalid compression level 23 for zstd. Expected a level between 0 and 22",+ ),+ (+ "gzip(10)",+ "Invalid compression level 10 for gzip. Expected a level between 0 and 9",+ ),+ ("brotli(12)", "Invalid compression level 12 for brotli"),+ ("", "Unknown or unsupported parquet compression: "),+ ] {+ let err = input+ .parse::<DFParquetCompression>()+ .unwrap_err()+ .to_string();+ assert!(err.contains(message), "input {input:?}: {err}");+ }+ }++ /// The level ranges above must stay equal to the ones parquet enforces, or+ /// the infallible conversion would panic on a level parquet rejects.+ #[cfg(feature = "parquet")]+ #[test]+ fn parquet_compression_level_ranges_match_parquet() {+ use parquet::basic::{BrotliLevel, GzipLevel, ZstdLevel};++ for level in GZIP_LEVELS {+ assert!(GzipLevel::try_new(level).is_ok(), "gzip({level})");+ }+ assert!(GzipLevel::try_new(GZIP_LEVELS.end() + 1).is_err());++ for level in BROTLI_LEVELS {+ assert!(BrotliLevel::try_new(level).is_ok(), "brotli({level})");+ }+ assert!(BrotliLevel::try_new(BROTLI_LEVELS.end() + 1).is_err());++ for level in ZSTD_LEVELS {+ assert!(ZstdLevel::try_new(level as i32).is_ok(), "zstd({level})");+ }+ assert!(ZstdLevel::try_new(*ZSTD_LEVELS.end() as i32 + 1).is_err());+ }+}diff --git a/datafusion/core/src/dataframe/parquet.rs b/datafusion/core/src/dataframe/parquet.rsindex 1685dff23d..dd751eb977 100644--- a/datafusion/core/src/dataframe/parquet.rs+++ b/datafusion/core/src/dataframe/parquet.rs@@ -189,7 +189,7 @@ mod tests {let ctx = &test_df.session_state;ctx.runtime_env().register_object_store(&local_url, local);let mut options = TableParquetOptions::default();- options.global.compression = Some(compression.to_string());+ options.global.compression = Some(compression.parse()?);df.write_parquet(output_path,DataFrameWriteOptions::new().with_single_file_output(true),diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 982766dc88..a620ff2f08 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -808,9 +808,10 @@ mod tests {).await.expect_err("Example should fail!");+ // The value is rejected when the option is set, before any file is writtenassert_eq!(e.strip_backtrace(),- "Invalid or Unsupported Configuration: zstd compression requires specifying a level such as zstd(4)"+ "Error setting config datafusion.execution.parquet.compression\ncaused by\nInvalid or Unsupported Configuration: zstd compression requires specifying a level such as zstd(4)");Ok(())diff --git a/datafusion/core/src/execution/context/parquet.rs b/datafusion/core/src/execution/context/parquet.rsindex aad8518ba4..c015b99c0d 100644--- a/datafusion/core/src/execution/context/parquet.rs+++ b/datafusion/core/src/execution/context/parquet.rs@@ -103,11 +103,11 @@ mod tests {use crate::arrow::datatypes::{DataType, Field, Schema};use crate::arrow::record_batch::RecordBatch;use crate::dataframe::DataFrameWriteOptions;- use crate::parquet::basic::Compression;use crate::test_util::parquet_test_data;use arrow::util::pretty::pretty_format_batches;use datafusion_common::config::TableParquetOptions;+ use datafusion_common::parquet_config::DFParquetCompression;use datafusion_common::{assert_batches_sorted_eq, assert_contains};use datafusion_execution::config::SessionConfig;@@ -271,7 +271,7 @@ mod tests {std::fs::create_dir(dir).expect("create dir failed");let mut options = TableParquetOptions::default();- options.global.compression = Some(Compression::SNAPPY.to_string());+ options.global.compression = Some(DFParquetCompression::Snappy);// Write the dataframe to a parquet file named 'output1.parquet'write_dfdiff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex 65ea0416b0..dc0f9a7be4 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -670,7 +670,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {write_batch_size: global_options.global.write_batch_size as u64,writer_version: global_options.global.writer_version.to_string(),compression_opt: global_options.global.compression.map(|compression| {- parquet_options::CompressionOpt::Compression(compression)+ parquet_options::CompressionOpt::Compression(compression.to_string())}),dictionary_enabled_opt: global_options.global.dictionary_enabled.map(|enabled| {parquet_options::DictionaryEnabledOpt::DictionaryEnabled(enabled)diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 62b16664f1..0bde696c7c 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -1202,9 +1202,9 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {writer_version: value.writer_version.parse().map_err(|e| {DataFusionError::Internal(format!("Failed to parse writer_version: {e}"))})?,- compression: value.compression_opt.clone().map(|opt| match opt {- protobuf::parquet_options::CompressionOpt::Compression(v) => Some(v),- }).unwrap_or(None),+ compression: value.compression_opt.as_ref().map(|opt| match opt {+ protobuf::parquet_options::CompressionOpt::Compression(v) => v.parse(),+ }).transpose()?,dictionary_enabled: value.dictionary_enabled_opt.as_ref().map(|protobuf::parquet_options::DictionaryEnabledOpt::DictionaryEnabled(v)| *v),// Continuing from where we left off in the TryFrom implementationdictionary_page_size_limit: to_usize(@@ -1438,7 +1438,7 @@ mod tests {use datafusion_common::config::{MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,};- use datafusion_common::parquet_config::DFParquetStatistics;+ use datafusion_common::parquet_config::{DFParquetCompression, DFParquetStatistics};#[test]fn constraint_requires_mode() {@@ -1561,6 +1561,35 @@ mod tests {);}+ #[test]+ fn test_parquet_compression_round_trip() {+ let opts = ParquetOptions {+ compression: Some(DFParquetCompression::Gzip(6)),+ ..ParquetOptions::default()+ };+ let recovered = parquet_options_proto_round_trip(opts);+ assert_eq!(recovered.compression, Some(DFParquetCompression::Gzip(6)));+ }++ #[test]+ fn test_invalid_parquet_compression_rejected_from_proto() {+ let opts = ParquetOptions::default();+ let mut proto: crate::protobuf_common::ParquetOptions =+ (&opts).try_into().expect("to_proto");+ proto.compression_opt = Some(+ crate::protobuf_common::parquet_options::CompressionOpt::Compression(+ "zstd".to_string(),+ ),+ );++ let err = ParquetOptions::try_from(&proto).unwrap_err();+ assert!(+ err.to_string()+ .contains("zstd compression requires specifying a level such as zstd(4)"),+ "unexpected error: {err}"+ );+ }+#[test]fn test_invalid_parquet_statistics_rejected_from_proto() {let opts = ParquetOptions::default();diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex f9a881e76d..b8eb46f2c8 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -942,7 +942,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {data_pagesize_limit: value.data_pagesize_limit as u64,write_batch_size: value.write_batch_size as u64,writer_version: value.writer_version.to_string(),- compression_opt: value.compression.clone().map(protobuf::parquet_options::CompressionOpt::Compression),+ compression_opt: value.compression.map(|v| protobuf::parquet_options::CompressionOpt::Compression(v.to_string())),dictionary_enabled_opt: value.dictionary_enabled.map(protobuf::parquet_options::DictionaryEnabledOpt::DictionaryEnabled),dictionary_page_size_limit: value.dictionary_page_size_limit as u64,statistics_enabled_opt: value.statistics_enabled.map(|v| protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v.to_string())),diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rsindex b2340a0967..fdf3239647 100644--- a/datafusion/proto-models/src/from_proto.rs+++ b/datafusion/proto-models/src/from_proto.rs@@ -380,11 +380,15 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {)?,write_batch_size: to_usize(proto.write_batch_size, "write_batch_size")?,writer_version,- compression: proto.compression_opt.as_ref().map(|opt| match opt {- parquet_options::CompressionOpt::Compression(compression) => {- compression.clone()- }- }),+ compression: proto+ .compression_opt+ .as_ref()+ .map(|opt| match opt {+ parquet_options::CompressionOpt::Compression(compression) => {+ compression.parse()+ }+ })+ .transpose()?,dictionary_enabled: proto.dictionary_enabled_opt.as_ref().map(|opt| {match opt {parquet_options::DictionaryEnabledOpt::DictionaryEnabled(diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex 5d9cc3bd4a..bf9145fe44 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -751,6 +751,48 @@ Invalid or Unsupported Configuration: Invalid parquet statistics setting: invalistatement okRESET datafusion.execution.parquet.statistics_enabled+# parquet.compression is validated when it is set, including the level+statement ok+SET datafusion.execution.parquet.compression = 'GZIP(6)'++query TT+SHOW datafusion.execution.parquet.compression+----+datafusion.execution.parquet.compression gzip(6)++statement error+SET datafusion.execution.parquet.compression = 'zstdd(3)'+----+DataFusion error: Error setting config datafusion.execution.parquet.compression+caused by+Invalid or Unsupported Configuration: Unknown or unsupported parquet compression: zstdd(3). Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw.+++statement error+SET datafusion.execution.parquet.compression = 'zstd'+----+DataFusion error: Error setting config datafusion.execution.parquet.compression+caused by+Invalid or Unsupported Configuration: zstd compression requires specifying a level such as zstd(4)+++statement error+SET datafusion.execution.parquet.compression = 'zstd(23)'+----+DataFusion error: Error setting config datafusion.execution.parquet.compression+caused by+Invalid or Unsupported Configuration: Invalid compression level 23 for zstd. Expected a level between 0 and 22+++# The rejected values did not change the setting+query TT+SHOW datafusion.execution.parquet.compression+----+datafusion.execution.parquet.compression gzip(6)++statement ok+RESET datafusion.execution.parquet.compression+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0SET datafusion.execution.batch_size = 0