Apache DataFusion SQL Query Engine
fix: validate parquet encoding when configured (#25784)
QQQ96eb0dfddaf3
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex b2cfdbfc2e..290baeb379 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -24,8 +24,8 @@ use crate::encryption::{FileDecryptionProperties, FileEncryptionProperties};use crate::error::{_config_datafusion_err, _config_err};use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};use crate::parquet_config::{- DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion, DFTimeUnit,- RowGroupRangeAssignment,+ DFParquetCompression, DFParquetEncoding, DFParquetStatistics, DFParquetWriterVersion,+ DFTimeUnit, RowGroupRangeAssignment,};use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};use crate::utils::get_available_parallelism;@@ -1544,7 +1544,7 @@ config_namespace! {/// delta_byte_array, rle_dictionary, and byte_stream_split./// These values are not case sensitive. If NULL, uses/// default parquet writer setting- pub encoding: Option<String>, transform = str::to_lowercase, default = None+ pub encoding: Option<DFParquetEncoding>, default = None/// (writing) Write bloom filters for all columns when creating parquet filespub bloom_filter_on_write: bool, default = falsediff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rsindex f6485c2f54..3d904290b2 100644--- a/datafusion/common/src/file_options/parquet_writer.rs+++ b/datafusion/common/src/file_options/parquet_writer.rs@@ -22,7 +22,7 @@ use std::sync::Arc;use crate::{_internal_datafusion_err, DataFusionError, Result,config::{ParquetCdcOptions, ParquetOptions, TableParquetOptions},- parquet_config::DFParquetCompression,+ parquet_config::{DFParquetCompression, DFParquetEncoding},};use arrow::datatypes::Schema;@@ -291,7 +291,7 @@ impl ParquetOptions {builder = builder.set_compression((*compression).into());}if let Some(encoding) = encoding {- builder = builder.set_encoding(parse_encoding_string(encoding)?);+ builder = builder.set_encoding((*encoding).into());}builder = builder.set_content_defined_chunking(content_defined_chunking.into());@@ -299,31 +299,11 @@ impl ParquetOptions {}}-/// Parses datafusion.execution.parquet.encoding String to a parquet::basic::Encoding+/// Parses a column-specific parquet encoding string.pub(crate) fn parse_encoding_string(str_setting: &str,) -> Result<parquet::basic::Encoding> {- let str_setting_lower: &str = &str_setting.to_lowercase();- match str_setting_lower {- "plain" => Ok(parquet::basic::Encoding::PLAIN),- "plain_dictionary" => Ok(parquet::basic::Encoding::PLAIN_DICTIONARY),- "rle" => Ok(parquet::basic::Encoding::RLE),- #[expect(deprecated)]- "bit_packed" => Ok(parquet::basic::Encoding::BIT_PACKED),- "delta_binary_packed" => Ok(parquet::basic::Encoding::DELTA_BINARY_PACKED),- "delta_length_byte_array" => {- Ok(parquet::basic::Encoding::DELTA_LENGTH_BYTE_ARRAY)- }- "delta_byte_array" => Ok(parquet::basic::Encoding::DELTA_BYTE_ARRAY),- "rle_dictionary" => Ok(parquet::basic::Encoding::RLE_DICTIONARY),- "byte_stream_split" => Ok(parquet::basic::Encoding::BYTE_STREAM_SPLIT),- _ => Err(DataFusionError::Configuration(format!(- "Unknown or unsupported parquet encoding: \- {str_setting}. Valid values are: plain, plain_dictionary, rle, \- bit_packed, delta_binary_packed, delta_length_byte_array, \- delta_byte_array, rle_dictionary, and byte_stream_split."- ))),- }+ str_setting.parse::<DFParquetEncoding>().map(Into::into)}/// Parses datafusion.execution.parquet.compression String to a parquet::basic::Compression@@ -357,7 +337,8 @@ mod tests {ParquetEncryptionOptions, ParquetOptions,};use crate::parquet_config::{- DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion,+ DFParquetCompression, DFParquetEncoding, DFParquetStatistics,+ DFParquetWriterVersion,};use parquet::basic::Compression;use parquet::file::properties::{@@ -406,7 +387,7 @@ mod tests {column_index_truncate_length: Some(42),statistics_truncate_length: Some(42),data_page_row_count_limit: 42,- encoding: Some("BYTE_STREAM_SPLIT".into()),+ encoding: Some(DFParquetEncoding::ByteStreamSplit),bloom_filter_on_write: !defaults.bloom_filter_on_write,bloom_filter_fpp: Some(0.42),bloom_filter_ndv: Some(42),@@ -525,7 +506,10 @@ mod tests {data_page_row_count_limit: props.data_page_row_count_limit(),// global options which set the default column props- encoding: default_col_props.encoding,+ encoding: default_col_props+ .encoding+ .as_deref()+ .map(|encoding| encoding.parse().expect("valid parquet encoding")),compression: match props.compression(&default_col) {Compression::ZSTD(level) => {Some(DFParquetCompression::Zstd(level.compression_level() as u32))diff --git a/datafusion/common/src/parquet_config.rs b/datafusion/common/src/parquet_config.rsindex cc466ed14e..9d0fa1b670 100644--- a/datafusion/common/src/parquet_config.rs+++ b/datafusion/common/src/parquet_config.rs@@ -274,6 +274,123 @@ impl From<parquet::file::properties::EnabledStatistics> for DFParquetStatistics}}+/// Parquet encodings accepted by the writer.+///+/// Parsing the encoding when it is configured rejects unsupported values before+/// a Parquet write begins.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DFParquetEncoding {+ /// Plain encoding+ Plain,+ /// Deprecated plain dictionary encoding+ PlainDictionary,+ /// Run-length encoding+ Rle,+ /// Deprecated bit-packed encoding+ BitPacked,+ /// Delta encoding for integers+ DeltaBinaryPacked,+ /// Delta length encoding for byte arrays+ DeltaLengthByteArray,+ /// Delta encoding for byte arrays+ DeltaByteArray,+ /// Run-length dictionary encoding+ RleDictionary,+ /// Byte stream split encoding+ ByteStreamSplit,+}++impl FromStr for DFParquetEncoding {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self, Self::Err> {+ match s.to_ascii_lowercase().as_str() {+ "plain" => Ok(Self::Plain),+ "plain_dictionary" => Ok(Self::PlainDictionary),+ "rle" => Ok(Self::Rle),+ "bit_packed" => Ok(Self::BitPacked),+ "delta_binary_packed" => Ok(Self::DeltaBinaryPacked),+ "delta_length_byte_array" => Ok(Self::DeltaLengthByteArray),+ "delta_byte_array" => Ok(Self::DeltaByteArray),+ "rle_dictionary" => Ok(Self::RleDictionary),+ "byte_stream_split" => Ok(Self::ByteStreamSplit),+ _ => Err(DataFusionError::Configuration(format!(+ "Unknown or unsupported parquet encoding: \+ {s}. Valid values are: plain, plain_dictionary, rle, \+ bit_packed, delta_binary_packed, delta_length_byte_array, \+ delta_byte_array, rle_dictionary, and byte_stream_split."+ ))),+ }+ }+}++impl Display for DFParquetEncoding {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(match self {+ Self::Plain => "plain",+ Self::PlainDictionary => "plain_dictionary",+ Self::Rle => "rle",+ Self::BitPacked => "bit_packed",+ Self::DeltaBinaryPacked => "delta_binary_packed",+ Self::DeltaLengthByteArray => "delta_length_byte_array",+ Self::DeltaByteArray => "delta_byte_array",+ Self::RleDictionary => "rle_dictionary",+ Self::ByteStreamSplit => "byte_stream_split",+ })+ }+}++impl ConfigField for Option<DFParquetEncoding> {+ fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {+ match self {+ Some(encoding) => v.some(key, encoding, description),+ None => v.none(key, description),+ }+ }++ fn set(&mut self, key: &str, value: &str) -> Result<()> {+ if !key.is_empty() {+ return crate::error::_config_err!(+ "Config field parquet.encoding is a scalar Option<DFParquetEncoding> and does not have nested field \"{}\"",+ key+ );+ }++ *self = Some(value.parse()?);+ Ok(())+ }++ fn reset(&mut self, key: &str) -> Result<()> {+ if key.is_empty() {+ *self = None;+ Ok(())+ } else {+ crate::error::_config_err!(+ "Config field parquet.encoding is a scalar Option<DFParquetEncoding> and does not have nested field \"{}\"",+ key+ )+ }+ }+}++#[cfg(feature = "parquet")]+impl From<DFParquetEncoding> for parquet::basic::Encoding {+ fn from(value: DFParquetEncoding) -> Self {+ match value {+ DFParquetEncoding::Plain => Self::PLAIN,+ DFParquetEncoding::PlainDictionary => Self::PLAIN_DICTIONARY,+ DFParquetEncoding::Rle => Self::RLE,+ #[expect(deprecated)]+ DFParquetEncoding::BitPacked => Self::BIT_PACKED,+ DFParquetEncoding::DeltaBinaryPacked => Self::DELTA_BINARY_PACKED,+ DFParquetEncoding::DeltaLengthByteArray => Self::DELTA_LENGTH_BYTE_ARRAY,+ DFParquetEncoding::DeltaByteArray => Self::DELTA_BYTE_ARRAY,+ DFParquetEncoding::RleDictionary => Self::RLE_DICTIONARY,+ DFParquetEncoding::ByteStreamSplit => Self::BYTE_STREAM_SPLIT,+ }+ }+}+/// Parquet compression codec, with a level for the codecs that take one////// This enum validates compression settings at configuration time. An unknown@@ -589,6 +706,67 @@ impl From<DFTimeUnit> for arrow::datatypes::TimeUnit {mod tests {use super::*;+ #[cfg(feature = "parquet")]+ #[test]+ fn parquet_encoding_parses_displays_and_converts() {+ use parquet::basic::Encoding;++ for (input, expected, parquet_encoding) in [+ ("PLAIN", DFParquetEncoding::Plain, Encoding::PLAIN),+ (+ "plain_dictionary",+ DFParquetEncoding::PlainDictionary,+ Encoding::PLAIN_DICTIONARY,+ ),+ ("rle", DFParquetEncoding::Rle, Encoding::RLE),+ #[expect(deprecated)]+ (+ "bit_packed",+ DFParquetEncoding::BitPacked,+ Encoding::BIT_PACKED,+ ),+ (+ "delta_binary_packed",+ DFParquetEncoding::DeltaBinaryPacked,+ Encoding::DELTA_BINARY_PACKED,+ ),+ (+ "delta_length_byte_array",+ DFParquetEncoding::DeltaLengthByteArray,+ Encoding::DELTA_LENGTH_BYTE_ARRAY,+ ),+ (+ "delta_byte_array",+ DFParquetEncoding::DeltaByteArray,+ Encoding::DELTA_BYTE_ARRAY,+ ),+ (+ "rle_dictionary",+ DFParquetEncoding::RleDictionary,+ Encoding::RLE_DICTIONARY,+ ),+ (+ "byte_stream_split",+ DFParquetEncoding::ByteStreamSplit,+ Encoding::BYTE_STREAM_SPLIT,+ ),+ ] {+ let parsed: DFParquetEncoding = input.parse().unwrap();+ assert_eq!(parsed, expected, "input: {input}");+ assert_eq!(parsed.to_string(), input.to_ascii_lowercase());+ assert_eq!(Encoding::from(parsed), parquet_encoding);+ }++ for input in ["", "invalid", "zstd"] {+ let err = input.parse::<DFParquetEncoding>().unwrap_err();+ assert!(+ err.to_string()+ .contains("Unknown or unsupported parquet encoding"),+ "input {input:?}: {err}"+ );+ }+ }+#[test]fn parquet_compression_parses_and_displays() {for (input, expected, displayed) in [diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex ca5a4bddec..50e76d2d21 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -739,7 +739,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {}),data_page_row_count_limit: global_options.global.data_page_row_count_limit as u64,encoding_opt: global_options.global.encoding.map(|encoding| {- parquet_options::EncodingOpt::Encoding(encoding)+ parquet_options::EncodingOpt::Encoding(encoding.to_string())}),bloom_filter_on_read: global_options.global.bloom_filter_on_read,bloom_filter_on_write: global_options.global.bloom_filter_on_write,diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 42b7c4976c..24429f2fd1 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -1253,11 +1253,12 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {"data_page_row_count_limit",)?,encoding: value- .encoding_opt.clone()+ .encoding_opt+ .as_ref().map(|opt| match opt {- protobuf::parquet_options::EncodingOpt::Encoding(v) => Some(v),+ protobuf::parquet_options::EncodingOpt::Encoding(v) => v.parse(),})- .unwrap_or(None),+ .transpose()?,bloom_filter_on_read: value.bloom_filter_on_read,bloom_filter_on_write: value.bloom_filter_on_write,bloom_filter_fpp: value.clone()@@ -1454,7 +1455,8 @@ mod tests {MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,};use datafusion_common::parquet_config::{- DFParquetCompression, DFParquetStatistics, DFTimeUnit, RowGroupRangeAssignment,+ DFParquetCompression, DFParquetEncoding, DFParquetStatistics, DFTimeUnit,+ RowGroupRangeAssignment,};#[test]@@ -1721,6 +1723,37 @@ mod tests {assert_eq!(recovered.compression, Some(DFParquetCompression::Gzip(6)));}+ #[test]+ fn test_parquet_encoding_round_trip() {+ for encoding in [None, Some(DFParquetEncoding::ByteStreamSplit)] {+ let opts = ParquetOptions {+ encoding,+ ..ParquetOptions::default()+ };+ let recovered = parquet_options_proto_round_trip(opts);+ assert_eq!(recovered.encoding, encoding);+ }+ }++ #[test]+ fn test_invalid_parquet_encoding_rejected_from_proto() {+ let opts = ParquetOptions::default();+ let mut proto: crate::protobuf_common::ParquetOptions =+ (&opts).try_into().expect("to_proto");+ proto.encoding_opt = Some(+ crate::protobuf_common::parquet_options::EncodingOpt::Encoding(+ "invalid".to_string(),+ ),+ );++ let err = ParquetOptions::try_from(&proto).unwrap_err();+ assert!(+ err.to_string()+ .contains("Unknown or unsupported parquet encoding: invalid"),+ "unexpected error: {err}"+ );+ }+#[test]fn test_invalid_parquet_compression_rejected_from_proto() {let opts = ParquetOptions::default();diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex 01f37a478f..22b291843d 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -955,7 +955,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {column_index_truncate_length_opt: value.column_index_truncate_length.map(|v| protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v as u64)),statistics_truncate_length_opt: value.statistics_truncate_length.map(|v| protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v as u64)),data_page_row_count_limit: value.data_page_row_count_limit as u64,- encoding_opt: value.encoding.clone().map(protobuf::parquet_options::EncodingOpt::Encoding),+ encoding_opt: value.encoding.map(|encoding| protobuf::parquet_options::EncodingOpt::Encoding(encoding.to_string())),bloom_filter_on_read: value.bloom_filter_on_read,bloom_filter_on_write: value.bloom_filter_on_write,bloom_filter_fpp_opt: value.bloom_filter_fpp.map(protobuf::parquet_options::BloomFilterFppOpt::BloomFilterFpp),diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rsindex cdf7c30d00..6d3cadc577 100644--- a/datafusion/proto-models/src/from_proto.rs+++ b/datafusion/proto-models/src/from_proto.rs@@ -439,11 +439,13 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {proto.data_page_row_count_limit,"data_page_row_count_limit",)?,- encoding: proto.encoding_opt.as_ref().map(|opt| match opt {- parquet_options::EncodingOpt::Encoding(encoding) => {- encoding.clone()- }- }),+ encoding: proto+ .encoding_opt+ .as_ref()+ .map(|opt| match opt {+ parquet_options::EncodingOpt::Encoding(encoding) => encoding.parse(),+ })+ .transpose()?,bloom_filter_on_read: proto.bloom_filter_on_read,bloom_filter_on_write: proto.bloom_filter_on_write,bloom_filter_fpp: proto@@ -583,6 +585,22 @@ impl TryFrom<&TableParquetOptionsProto> for TableParquetOptions {mod tests {use super::*;+ #[test]+ fn rejects_invalid_parquet_encoding() {+ let proto = ParquetOptionsProto {+ encoding_opt: Some(parquet_options::EncodingOpt::Encoding(+ "invalid".to_string(),+ )),+ ..Default::default()+ };++ let err = ParquetOptions::try_from(&proto).unwrap_err();+ assert!(+ err.to_string()+ .contains("Unknown or unsupported parquet encoding: invalid")+ );+ }+#[test]fn rejects_invalid_parquet_statistics() {let proto = ParquetOptionsProto {diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex 2ea93fcefa..307de8c24c 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -758,6 +758,31 @@ Invalid or Unsupported Configuration: Invalid parquet statistics setting: invalistatement okRESET datafusion.execution.parquet.statistics_enabled+# parquet.encoding is validated when it is set+statement ok+SET datafusion.execution.parquet.encoding = 'PLAIN'++query TT+SHOW datafusion.execution.parquet.encoding+----+datafusion.execution.parquet.encoding plain++statement error+SET datafusion.execution.parquet.encoding = 'invalid'+----+DataFusion error: Error setting config datafusion.execution.parquet.encoding+caused by+Invalid or Unsupported Configuration: Unknown or unsupported parquet encoding: invalid. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split.+++query TT+SHOW datafusion.execution.parquet.encoding+----+datafusion.execution.parquet.encoding plain++statement ok+RESET datafusion.execution.parquet.encoding+# parquet.compression is validated when it is set, including the levelstatement okSET datafusion.execution.parquet.compression = 'GZIP(6)'