Skip to content

Apache DataFusion SQL Query Engine

fix: validate parquet encoding when configured (#25784)

QQQ96eb0dfddaf3
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index b2cfdbfc2e..290baeb379 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -24,8 +24,8 @@ use crate::encryption::{FileDecryptionProperties, FileEncryptionProperties};
use crate::error::{_config_datafusion_err, _config_err};
use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};
use crate::parquet_config::{
- DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion, DFTimeUnit,
- RowGroupRangeAssignment,
+ DFParquetCompression, DFParquetEncoding, DFParquetStatistics, DFParquetWriterVersion,
+ DFTimeUnit, RowGroupRangeAssignment,
};
use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};
use crate::utils::get_available_parallelism;
@@ -1544,7 +1544,7 @@ config_namespace! {
/// delta_byte_array, rle_dictionary, and byte_stream_split.
/// These values are not case sensitive. If NULL, uses
/// default parquet writer setting
- pub encoding: Option<String>, transform = str::to_lowercase, default = None
+ pub encoding: Option<DFParquetEncoding>, default = None
/// (writing) Write bloom filters for all columns when creating parquet files
pub bloom_filter_on_write: bool, default = false
diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs
index f6485c2f54..3d904290b2 100644
--- a/datafusion/common/src/file_options/parquet_writer.rs
+++ b/datafusion/common/src/file_options/parquet_writer.rs
@@ -22,7 +22,7 @@ use std::sync::Arc;
use crate::{
_internal_datafusion_err, DataFusionError, Result,
config::{ParquetCdcOptions, ParquetOptions, TableParquetOptions},
- parquet_config::DFParquetCompression,
+ parquet_config::{DFParquetCompression, DFParquetEncoding},
};
use arrow::datatypes::Schema;
@@ -291,7 +291,7 @@ impl ParquetOptions {
builder = builder.set_compression((*compression).into());
}
if let Some(encoding) = encoding {
- builder = builder.set_encoding(parse_encoding_string(encoding)?);
+ builder = builder.set_encoding((*encoding).into());
}
builder = builder.set_content_defined_chunking(content_defined_chunking.into());
@@ -299,31 +299,11 @@ impl ParquetOptions {
}
}
-/// Parses datafusion.execution.parquet.encoding String to a parquet::basic::Encoding
+/// Parses a column-specific parquet encoding string.
pub(crate) fn parse_encoding_string(
str_setting: &str,
) -> Result<parquet::basic::Encoding> {
- let str_setting_lower: &str = &str_setting.to_lowercase();
- match str_setting_lower {
- "plain" => Ok(parquet::basic::Encoding::PLAIN),
- "plain_dictionary" => Ok(parquet::basic::Encoding::PLAIN_DICTIONARY),
- "rle" => Ok(parquet::basic::Encoding::RLE),
- #[expect(deprecated)]
- "bit_packed" => Ok(parquet::basic::Encoding::BIT_PACKED),
- "delta_binary_packed" => Ok(parquet::basic::Encoding::DELTA_BINARY_PACKED),
- "delta_length_byte_array" => {
- Ok(parquet::basic::Encoding::DELTA_LENGTH_BYTE_ARRAY)
- }
- "delta_byte_array" => Ok(parquet::basic::Encoding::DELTA_BYTE_ARRAY),
- "rle_dictionary" => Ok(parquet::basic::Encoding::RLE_DICTIONARY),
- "byte_stream_split" => Ok(parquet::basic::Encoding::BYTE_STREAM_SPLIT),
- _ => Err(DataFusionError::Configuration(format!(
- "Unknown or unsupported parquet encoding: \
- {str_setting}. Valid values are: plain, plain_dictionary, rle, \
- bit_packed, delta_binary_packed, delta_length_byte_array, \
- delta_byte_array, rle_dictionary, and byte_stream_split."
- ))),
- }
+ str_setting.parse::<DFParquetEncoding>().map(Into::into)
}
/// Parses datafusion.execution.parquet.compression String to a parquet::basic::Compression
@@ -357,7 +337,8 @@ mod tests {
ParquetEncryptionOptions, ParquetOptions,
};
use crate::parquet_config::{
- DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion,
+ DFParquetCompression, DFParquetEncoding, DFParquetStatistics,
+ DFParquetWriterVersion,
};
use parquet::basic::Compression;
use parquet::file::properties::{
@@ -406,7 +387,7 @@ mod tests {
column_index_truncate_length: Some(42),
statistics_truncate_length: Some(42),
data_page_row_count_limit: 42,
- encoding: Some("BYTE_STREAM_SPLIT".into()),
+ encoding: Some(DFParquetEncoding::ByteStreamSplit),
bloom_filter_on_write: !defaults.bloom_filter_on_write,
bloom_filter_fpp: Some(0.42),
bloom_filter_ndv: Some(42),
@@ -525,7 +506,10 @@ mod tests {
data_page_row_count_limit: props.data_page_row_count_limit(),
// global options which set the default column props
- encoding: default_col_props.encoding,
+ encoding: default_col_props
+ .encoding
+ .as_deref()
+ .map(|encoding| encoding.parse().expect("valid parquet encoding")),
compression: match props.compression(&default_col) {
Compression::ZSTD(level) => {
Some(DFParquetCompression::Zstd(level.compression_level() as u32))
diff --git a/datafusion/common/src/parquet_config.rs b/datafusion/common/src/parquet_config.rs
index cc466ed14e..9d0fa1b670 100644
--- a/datafusion/common/src/parquet_config.rs
+++ b/datafusion/common/src/parquet_config.rs
@@ -274,6 +274,123 @@ impl From<parquet::file::properties::EnabledStatistics> for DFParquetStatistics
}
}
+/// Parquet encodings accepted by the writer.
+///
+/// Parsing the encoding when it is configured rejects unsupported values before
+/// a Parquet write begins.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DFParquetEncoding {
+ /// Plain encoding
+ Plain,
+ /// Deprecated plain dictionary encoding
+ PlainDictionary,
+ /// Run-length encoding
+ Rle,
+ /// Deprecated bit-packed encoding
+ BitPacked,
+ /// Delta encoding for integers
+ DeltaBinaryPacked,
+ /// Delta length encoding for byte arrays
+ DeltaLengthByteArray,
+ /// Delta encoding for byte arrays
+ DeltaByteArray,
+ /// Run-length dictionary encoding
+ RleDictionary,
+ /// Byte stream split encoding
+ ByteStreamSplit,
+}
+
+impl FromStr for DFParquetEncoding {
+ type Err = DataFusionError;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s.to_ascii_lowercase().as_str() {
+ "plain" => Ok(Self::Plain),
+ "plain_dictionary" => Ok(Self::PlainDictionary),
+ "rle" => Ok(Self::Rle),
+ "bit_packed" => Ok(Self::BitPacked),
+ "delta_binary_packed" => Ok(Self::DeltaBinaryPacked),
+ "delta_length_byte_array" => Ok(Self::DeltaLengthByteArray),
+ "delta_byte_array" => Ok(Self::DeltaByteArray),
+ "rle_dictionary" => Ok(Self::RleDictionary),
+ "byte_stream_split" => Ok(Self::ByteStreamSplit),
+ _ => Err(DataFusionError::Configuration(format!(
+ "Unknown or unsupported parquet encoding: \
+ {s}. Valid values are: plain, plain_dictionary, rle, \
+ bit_packed, delta_binary_packed, delta_length_byte_array, \
+ delta_byte_array, rle_dictionary, and byte_stream_split."
+ ))),
+ }
+ }
+}
+
+impl Display for DFParquetEncoding {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str(match self {
+ Self::Plain => "plain",
+ Self::PlainDictionary => "plain_dictionary",
+ Self::Rle => "rle",
+ Self::BitPacked => "bit_packed",
+ Self::DeltaBinaryPacked => "delta_binary_packed",
+ Self::DeltaLengthByteArray => "delta_length_byte_array",
+ Self::DeltaByteArray => "delta_byte_array",
+ Self::RleDictionary => "rle_dictionary",
+ Self::ByteStreamSplit => "byte_stream_split",
+ })
+ }
+}
+
+impl ConfigField for Option<DFParquetEncoding> {
+ fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
+ match self {
+ Some(encoding) => v.some(key, encoding, description),
+ None => v.none(key, description),
+ }
+ }
+
+ fn set(&mut self, key: &str, value: &str) -> Result<()> {
+ if !key.is_empty() {
+ return crate::error::_config_err!(
+ "Config field parquet.encoding is a scalar Option<DFParquetEncoding> and does not have nested field \"{}\"",
+ key
+ );
+ }
+
+ *self = Some(value.parse()?);
+ Ok(())
+ }
+
+ fn reset(&mut self, key: &str) -> Result<()> {
+ if key.is_empty() {
+ *self = None;
+ Ok(())
+ } else {
+ crate::error::_config_err!(
+ "Config field parquet.encoding is a scalar Option<DFParquetEncoding> and does not have nested field \"{}\"",
+ key
+ )
+ }
+ }
+}
+
+#[cfg(feature = "parquet")]
+impl From<DFParquetEncoding> for parquet::basic::Encoding {
+ fn from(value: DFParquetEncoding) -> Self {
+ match value {
+ DFParquetEncoding::Plain => Self::PLAIN,
+ DFParquetEncoding::PlainDictionary => Self::PLAIN_DICTIONARY,
+ DFParquetEncoding::Rle => Self::RLE,
+ #[expect(deprecated)]
+ DFParquetEncoding::BitPacked => Self::BIT_PACKED,
+ DFParquetEncoding::DeltaBinaryPacked => Self::DELTA_BINARY_PACKED,
+ DFParquetEncoding::DeltaLengthByteArray => Self::DELTA_LENGTH_BYTE_ARRAY,
+ DFParquetEncoding::DeltaByteArray => Self::DELTA_BYTE_ARRAY,
+ DFParquetEncoding::RleDictionary => Self::RLE_DICTIONARY,
+ DFParquetEncoding::ByteStreamSplit => Self::BYTE_STREAM_SPLIT,
+ }
+ }
+}
+
/// Parquet compression codec, with a level for the codecs that take one
///
/// This enum validates compression settings at configuration time. An unknown
@@ -589,6 +706,67 @@ impl From<DFTimeUnit> for arrow::datatypes::TimeUnit {
mod tests {
use super::*;
+ #[cfg(feature = "parquet")]
+ #[test]
+ fn parquet_encoding_parses_displays_and_converts() {
+ use parquet::basic::Encoding;
+
+ for (input, expected, parquet_encoding) in [
+ ("PLAIN", DFParquetEncoding::Plain, Encoding::PLAIN),
+ (
+ "plain_dictionary",
+ DFParquetEncoding::PlainDictionary,
+ Encoding::PLAIN_DICTIONARY,
+ ),
+ ("rle", DFParquetEncoding::Rle, Encoding::RLE),
+ #[expect(deprecated)]
+ (
+ "bit_packed",
+ DFParquetEncoding::BitPacked,
+ Encoding::BIT_PACKED,
+ ),
+ (
+ "delta_binary_packed",
+ DFParquetEncoding::DeltaBinaryPacked,
+ Encoding::DELTA_BINARY_PACKED,
+ ),
+ (
+ "delta_length_byte_array",
+ DFParquetEncoding::DeltaLengthByteArray,
+ Encoding::DELTA_LENGTH_BYTE_ARRAY,
+ ),
+ (
+ "delta_byte_array",
+ DFParquetEncoding::DeltaByteArray,
+ Encoding::DELTA_BYTE_ARRAY,
+ ),
+ (
+ "rle_dictionary",
+ DFParquetEncoding::RleDictionary,
+ Encoding::RLE_DICTIONARY,
+ ),
+ (
+ "byte_stream_split",
+ DFParquetEncoding::ByteStreamSplit,
+ Encoding::BYTE_STREAM_SPLIT,
+ ),
+ ] {
+ let parsed: DFParquetEncoding = input.parse().unwrap();
+ assert_eq!(parsed, expected, "input: {input}");
+ assert_eq!(parsed.to_string(), input.to_ascii_lowercase());
+ assert_eq!(Encoding::from(parsed), parquet_encoding);
+ }
+
+ for input in ["", "invalid", "zstd"] {
+ let err = input.parse::<DFParquetEncoding>().unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("Unknown or unsupported parquet encoding"),
+ "input {input:?}: {err}"
+ );
+ }
+ }
+
#[test]
fn parquet_compression_parses_and_displays() {
for (input, expected, displayed) in [
diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs
index ca5a4bddec..50e76d2d21 100644
--- a/datafusion/datasource-parquet/src/file_format.rs
+++ b/datafusion/datasource-parquet/src/file_format.rs
@@ -739,7 +739,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
}),
data_page_row_count_limit: global_options.global.data_page_row_count_limit as u64,
encoding_opt: global_options.global.encoding.map(|encoding| {
- parquet_options::EncodingOpt::Encoding(encoding)
+ parquet_options::EncodingOpt::Encoding(encoding.to_string())
}),
bloom_filter_on_read: global_options.global.bloom_filter_on_read,
bloom_filter_on_write: global_options.global.bloom_filter_on_write,
diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs
index 42b7c4976c..24429f2fd1 100644
--- a/datafusion/proto-common/src/from_proto/mod.rs
+++ b/datafusion/proto-common/src/from_proto/mod.rs
@@ -1253,11 +1253,12 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
"data_page_row_count_limit",
)?,
encoding: value
- .encoding_opt.clone()
+ .encoding_opt
+ .as_ref()
.map(|opt| match opt {
- protobuf::parquet_options::EncodingOpt::Encoding(v) => Some(v),
+ protobuf::parquet_options::EncodingOpt::Encoding(v) => v.parse(),
})
- .unwrap_or(None),
+ .transpose()?,
bloom_filter_on_read: value.bloom_filter_on_read,
bloom_filter_on_write: value.bloom_filter_on_write,
bloom_filter_fpp: value.clone()
@@ -1454,7 +1455,8 @@ mod tests {
MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,
};
use datafusion_common::parquet_config::{
- DFParquetCompression, DFParquetStatistics, DFTimeUnit, RowGroupRangeAssignment,
+ DFParquetCompression, DFParquetEncoding, DFParquetStatistics, DFTimeUnit,
+ RowGroupRangeAssignment,
};
#[test]
@@ -1721,6 +1723,37 @@ mod tests {
assert_eq!(recovered.compression, Some(DFParquetCompression::Gzip(6)));
}
+ #[test]
+ fn test_parquet_encoding_round_trip() {
+ for encoding in [None, Some(DFParquetEncoding::ByteStreamSplit)] {
+ let opts = ParquetOptions {
+ encoding,
+ ..ParquetOptions::default()
+ };
+ let recovered = parquet_options_proto_round_trip(opts);
+ assert_eq!(recovered.encoding, encoding);
+ }
+ }
+
+ #[test]
+ fn test_invalid_parquet_encoding_rejected_from_proto() {
+ let opts = ParquetOptions::default();
+ let mut proto: crate::protobuf_common::ParquetOptions =
+ (&opts).try_into().expect("to_proto");
+ proto.encoding_opt = Some(
+ crate::protobuf_common::parquet_options::EncodingOpt::Encoding(
+ "invalid".to_string(),
+ ),
+ );
+
+ let err = ParquetOptions::try_from(&proto).unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("Unknown or unsupported parquet encoding: invalid"),
+ "unexpected error: {err}"
+ );
+ }
+
#[test]
fn test_invalid_parquet_compression_rejected_from_proto() {
let opts = ParquetOptions::default();
diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rs
index 01f37a478f..22b291843d 100644
--- a/datafusion/proto-common/src/to_proto/mod.rs
+++ b/datafusion/proto-common/src/to_proto/mod.rs
@@ -955,7 +955,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {
column_index_truncate_length_opt: value.column_index_truncate_length.map(|v| protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v as u64)),
statistics_truncate_length_opt: value.statistics_truncate_length.map(|v| protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v as u64)),
data_page_row_count_limit: value.data_page_row_count_limit as u64,
- encoding_opt: value.encoding.clone().map(protobuf::parquet_options::EncodingOpt::Encoding),
+ encoding_opt: value.encoding.map(|encoding| protobuf::parquet_options::EncodingOpt::Encoding(encoding.to_string())),
bloom_filter_on_read: value.bloom_filter_on_read,
bloom_filter_on_write: value.bloom_filter_on_write,
bloom_filter_fpp_opt: value.bloom_filter_fpp.map(protobuf::parquet_options::BloomFilterFppOpt::BloomFilterFpp),
diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rs
index cdf7c30d00..6d3cadc577 100644
--- a/datafusion/proto-models/src/from_proto.rs
+++ b/datafusion/proto-models/src/from_proto.rs
@@ -439,11 +439,13 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
proto.data_page_row_count_limit,
"data_page_row_count_limit",
)?,
- encoding: proto.encoding_opt.as_ref().map(|opt| match opt {
- parquet_options::EncodingOpt::Encoding(encoding) => {
- encoding.clone()
- }
- }),
+ encoding: proto
+ .encoding_opt
+ .as_ref()
+ .map(|opt| match opt {
+ parquet_options::EncodingOpt::Encoding(encoding) => encoding.parse(),
+ })
+ .transpose()?,
bloom_filter_on_read: proto.bloom_filter_on_read,
bloom_filter_on_write: proto.bloom_filter_on_write,
bloom_filter_fpp: proto
@@ -583,6 +585,22 @@ impl TryFrom<&TableParquetOptionsProto> for TableParquetOptions {
mod tests {
use super::*;
+ #[test]
+ fn rejects_invalid_parquet_encoding() {
+ let proto = ParquetOptionsProto {
+ encoding_opt: Some(parquet_options::EncodingOpt::Encoding(
+ "invalid".to_string(),
+ )),
+ ..Default::default()
+ };
+
+ let err = ParquetOptions::try_from(&proto).unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("Unknown or unsupported parquet encoding: invalid")
+ );
+ }
+
#[test]
fn rejects_invalid_parquet_statistics() {
let proto = ParquetOptionsProto {
diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.slt
index 2ea93fcefa..307de8c24c 100644
--- a/datafusion/sqllogictest/test_files/set_variable.slt
+++ b/datafusion/sqllogictest/test_files/set_variable.slt
@@ -758,6 +758,31 @@ Invalid or Unsupported Configuration: Invalid parquet statistics setting: invali
statement ok
RESET datafusion.execution.parquet.statistics_enabled
+# parquet.encoding is validated when it is set
+statement ok
+SET datafusion.execution.parquet.encoding = 'PLAIN'
+
+query TT
+SHOW datafusion.execution.parquet.encoding
+----
+datafusion.execution.parquet.encoding plain
+
+statement error
+SET datafusion.execution.parquet.encoding = 'invalid'
+----
+DataFusion error: Error setting config datafusion.execution.parquet.encoding
+caused by
+Invalid or Unsupported Configuration: Unknown or unsupported parquet encoding: invalid. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split.
+
+
+query TT
+SHOW datafusion.execution.parquet.encoding
+----
+datafusion.execution.parquet.encoding plain
+
+statement ok
+RESET datafusion.execution.parquet.encoding
+
# parquet.compression is validated when it is set, including the level
statement ok
SET datafusion.execution.parquet.compression = 'GZIP(6)'