Apache DataFusion SQL Query Engine
fix: validate parquet statistics config (#24642)
Xinyao Zhang999c04cf84f9
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 6065957731..05bec60a78 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -23,7 +23,7 @@ use arrow_ipc::CompressionType;use crate::encryption::{FileDecryptionProperties, FileEncryptionProperties};use crate::error::{_config_datafusion_err, _config_err};use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};-use crate::parquet_config::DFParquetWriterVersion;+use crate::parquet_config::{DFParquetStatistics, DFParquetWriterVersion};use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};use crate::utils::get_available_parallelism;use crate::{DataFusionError, Result};@@ -1445,7 +1445,7 @@ config_namespace! {/// Valid values are: "none", "chunk", and "page"/// These values are not case sensitive. If NULL, uses/// default parquet writer setting- pub statistics_enabled: Option<String>, transform = str::to_lowercase, default = Some("page".into())+ pub statistics_enabled: Option<DFParquetStatistics>, default = Some(DFParquetStatistics::Page)/// (writing) Target maximum number of rows in each row group (defaults to 1M/// rows). Writing larger row groups requires more memory to write, but@@ -4609,6 +4609,73 @@ mod tests {);}+ #[cfg(feature = "parquet")]+ #[test]+ fn test_parquet_statistics_validation() {+ use crate::{config::ConfigOptions, parquet_config::DFParquetStatistics};++ let mut config = ConfigOptions::default();++ for (value, expected) in [+ ("none", DFParquetStatistics::None),+ ("CHUNK", DFParquetStatistics::Chunk),+ ("page", DFParquetStatistics::Page),+ ] {+ config+ .set("datafusion.execution.parquet.statistics_enabled", value)+ .unwrap();+ assert_eq!(config.execution.parquet.statistics_enabled, Some(expected));+ }++ let err = config+ .set("datafusion.execution.parquet.statistics_enabled", "invalid")+ .unwrap_err();+ assert_contains!(+ err.to_string(),+ "Invalid parquet statistics setting: invalid. Expected one of: none, chunk, page"+ );++ // An unset value can arise from deserialization. An invalid update must+ // leave that state unchanged rather than inserting the default.+ config.execution.parquet.statistics_enabled = None;+ assert_eq!(config.execution.parquet.statistics_enabled, None);++ assert!(+ config+ .set("datafusion.execution.parquet.statistics_enabled", "invalid")+ .is_err()+ );+ assert_eq!(config.execution.parquet.statistics_enabled, None);++ config.execution.parquet.statistics_enabled = Some(DFParquetStatistics::Page);+ assert!(+ config+ .set(+ "datafusion.execution.parquet.statistics_enabled.typo",+ "none"+ )+ .is_err()+ );+ assert_eq!(+ config.execution.parquet.statistics_enabled,+ Some(DFParquetStatistics::Page)+ );++ assert!(+ config+ .reset("datafusion.execution.parquet.statistics_enabled.typo")+ .is_err()+ );+ assert_eq!(+ config.execution.parquet.statistics_enabled,+ Some(DFParquetStatistics::Page)+ );++ let mut scalar = DFParquetStatistics::Page;+ assert!(ConfigField::set(&mut scalar, "typo", "none").is_err());+ assert_eq!(scalar, DFParquetStatistics::Page);+ }+#[cfg(feature = "parquet")]#[test]fn set_cdc_enabled_flag() {diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rsindex 78c14da960..c50bb42a38 100644--- a/datafusion/common/src/file_options/parquet_writer.rs+++ b/datafusion/common/src/file_options/parquet_writer.rs@@ -257,9 +257,8 @@ impl ParquetOptions {.set_writer_version((*writer_version).into()).set_dictionary_page_size_limit(*dictionary_page_size_limit).set_statistics_enabled(- statistics_enabled- .as_ref()- .and_then(|s| parse_statistics_string(s).ok())+ (*statistics_enabled)+ .map(Into::into).unwrap_or(DEFAULT_STATISTICS_ENABLED),).set_max_row_group_row_count(Some(*max_row_group_size))@@ -434,7 +433,7 @@ mod tests {MaxRowGroupBytes, ParquetCdcOptions, ParquetColumnOptions,ParquetEncryptionOptions, ParquetOptions,};- use crate::parquet_config::DFParquetWriterVersion;+ use crate::parquet_config::{DFParquetStatistics, DFParquetWriterVersion};use parquet::basic::Compression;use parquet::file::properties::{BloomFilterProperties, DEFAULT_BLOOM_FILTER_FPP, DEFAULT_BLOOM_FILTER_NDV,@@ -475,7 +474,7 @@ mod tests {compression: Some("zstd(22)".into()),dictionary_enabled: Some(!defaults.dictionary_enabled.unwrap_or(false)),dictionary_page_size_limit: 43,- statistics_enabled: Some("chunk".into()),+ statistics_enabled: Some(DFParquetStatistics::Chunk),max_row_group_size: 42,max_row_group_bytes: Some(MaxRowGroupBytes::try_new(42).unwrap()),created_by: "wordy".into(),@@ -545,7 +544,7 @@ mod tests {/// (use identity to confirm correct.)fn session_config_from_writer_props(props: &WriterProperties) -> TableParquetOptions {let default_col = ColumnPath::from("col doesn't have specific config");- let default_col_props = extract_column_options(props, default_col);+ let default_col_props = extract_column_options(props, default_col.clone());let configured_col = ColumnPath::from(COL_NAME);let configured_col_props = extract_column_options(props, configured_col);@@ -600,7 +599,7 @@ mod tests {encoding: default_col_props.encoding,compression: default_col_props.compression,dictionary_enabled: default_col_props.dictionary_enabled,- statistics_enabled: default_col_props.statistics_enabled,+ statistics_enabled: Some(props.statistics_enabled(&default_col).into()),bloom_filter_on_write: default_col_props.bloom_filter_enabled.unwrap_or_default(),diff --git a/datafusion/common/src/parquet_config.rs b/datafusion/common/src/parquet_config.rsindex 9d6d7a8856..b727664e77 100644--- a/datafusion/common/src/parquet_config.rs+++ b/datafusion/common/src/parquet_config.rs@@ -106,3 +106,118 @@ impl From<parquet::file::properties::WriterVersion> for DFParquetWriterVersion {}}}++/// Parquet statistics levels supported by the writer+///+/// This enum validates statistics settings at configuration time, ensuring only+/// `none`, `chunk`, or `page` can be set via `SET` commands or deserialization.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DFParquetStatistics {+ /// Do not write statistics+ None,+ /// Write chunk-level statistics+ Chunk,+ /// Write page-level statistics+ Page,+}++impl FromStr for DFParquetStatistics {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self, Self::Err> {+ match s.to_lowercase().as_str() {+ "none" => Ok(Self::None),+ "chunk" => Ok(Self::Chunk),+ "page" => Ok(Self::Page),+ other => Err(DataFusionError::Configuration(format!(+ "Invalid parquet statistics setting: {other}. Expected one of: none, chunk, page"+ ))),+ }+ }+}++impl Display for DFParquetStatistics {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ let s = match self {+ Self::None => "none",+ Self::Chunk => "chunk",+ Self::Page => "page",+ };+ f.write_str(s)+ }+}++impl ConfigField for DFParquetStatistics {+ fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {+ v.some(key, self, description)+ }++ fn set(&mut self, key: &str, value: &str) -> Result<()> {+ if !key.is_empty() {+ return crate::error::_config_err!(+ "Config field parquet.statistics_enabled is a scalar DFParquetStatistics and does not have nested field \"{}\"",+ key+ );+ }++ *self = Self::from_str(value)?;+ Ok(())+ }+}++/// `ConfigField` for `Option<DFParquetStatistics>` parses before assigning so+/// an invalid value does not turn an unset option into the default.+impl ConfigField for Option<DFParquetStatistics> {+ fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {+ match self {+ Some(statistics) => statistics.visit(v, key, description),+ None => v.none(key, description),+ }+ }++ fn set(&mut self, key: &str, value: &str) -> Result<()> {+ if !key.is_empty() {+ return crate::error::_config_err!(+ "Config field parquet.statistics_enabled is a scalar Option<DFParquetStatistics> and does not have nested field \"{}\"",+ key+ );+ }++ *self = Some(DFParquetStatistics::from_str(value)?);+ Ok(())+ }++ fn reset(&mut self, key: &str) -> Result<()> {+ if key.is_empty() {+ *self = None;+ Ok(())+ } else {+ crate::error::_config_err!(+ "Config field parquet.statistics_enabled is a scalar Option<DFParquetStatistics> and does not have nested field \"{}\"",+ key+ )+ }+ }+}++#[cfg(feature = "parquet")]+impl From<DFParquetStatistics> for parquet::file::properties::EnabledStatistics {+ fn from(value: DFParquetStatistics) -> Self {+ match value {+ DFParquetStatistics::None => Self::None,+ DFParquetStatistics::Chunk => Self::Chunk,+ DFParquetStatistics::Page => Self::Page,+ }+ }+}++#[cfg(feature = "parquet")]+impl From<parquet::file::properties::EnabledStatistics> for DFParquetStatistics {+ fn from(value: parquet::file::properties::EnabledStatistics) -> Self {+ match value {+ parquet::file::properties::EnabledStatistics::None => Self::None,+ parquet::file::properties::EnabledStatistics::Chunk => Self::Chunk,+ parquet::file::properties::EnabledStatistics::Page => Self::Page,+ }+ }+}diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex 1b25a2c632..1c6c875f01 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -743,7 +743,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {}),dictionary_page_size_limit: global_options.global.dictionary_page_size_limit as u64,statistics_enabled_opt: global_options.global.statistics_enabled.map(|enabled| {- parquet_options::StatisticsEnabledOpt::StatisticsEnabled(enabled)+ parquet_options::StatisticsEnabledOpt::StatisticsEnabled(enabled.to_string())}),max_row_group_size: global_options.global.max_row_group_size as u64,max_in_list_size: global_options.global.max_in_list_size as u64,diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 0f176ced0d..92506ad92b 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -1103,11 +1103,11 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {"dictionary_page_size_limit",)?,statistics_enabled: value- .statistics_enabled_opt.clone()+ .statistics_enabled_opt.as_ref().map(|opt| match opt {- protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v) => Some(v),+ protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v) => v.parse(),})- .unwrap_or(None),+ .transpose()?,max_row_group_size: to_usize(value.max_row_group_size, "max_row_group_size")?,max_in_list_size: to_usize(value.max_in_list_size, "max_in_list_size")?,created_by: value.created_by.clone(),@@ -1389,6 +1389,7 @@ mod tests {use datafusion_common::config::{MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,};+ use datafusion_common::parquet_config::DFParquetStatistics;#[test]fn constraint_requires_mode() {@@ -1498,6 +1499,39 @@ mod tests {assert_eq!(recovered.coerce_int96_tz, Some("UTC".to_string()));}+ #[test]+ fn test_parquet_statistics_round_trip() {+ let opts = ParquetOptions {+ statistics_enabled: Some(DFParquetStatistics::Chunk),+ ..ParquetOptions::default()+ };+ let recovered = parquet_options_proto_round_trip(opts);+ assert_eq!(+ recovered.statistics_enabled,+ Some(DFParquetStatistics::Chunk)+ );+ }++ #[test]+ fn test_invalid_parquet_statistics_rejected_from_proto() {+ let opts = ParquetOptions::default();+ let mut proto: crate::protobuf_common::ParquetOptions =+ (&opts).try_into().expect("to_proto");+ proto.statistics_enabled_opt = Some(+ crate::protobuf_common::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(+ "invalid".to_string(),+ ),+ );++ let err = ParquetOptions::try_from(&proto).unwrap_err();+ assert!(+ err.to_string().contains(+ "Invalid parquet statistics setting: invalid. Expected one of: none, chunk, page"+ ),+ "unexpected error: {err}"+ );+ }+#[test]fn test_parquet_options_max_row_group_bytes_round_trip() {let opts = ParquetOptions {diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex ae220a53d8..7f95e03f41 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -917,7 +917,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {compression_opt: value.compression.clone().map(protobuf::parquet_options::CompressionOpt::Compression),dictionary_enabled_opt: value.dictionary_enabled.map(protobuf::parquet_options::DictionaryEnabledOpt::DictionaryEnabled),dictionary_page_size_limit: value.dictionary_page_size_limit as u64,- statistics_enabled_opt: value.statistics_enabled.clone().map(protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled),+ statistics_enabled_opt: value.statistics_enabled.map(|v| protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v.to_string())),max_row_group_size: value.max_row_group_size as u64,max_in_list_size: value.max_in_list_size as u64,created_by: value.created_by.clone(),diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rsindex 16903ec388..b2340a0967 100644--- a/datafusion/proto-models/src/from_proto.rs+++ b/datafusion/proto-models/src/from_proto.rs@@ -396,13 +396,15 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {proto.dictionary_page_size_limit,"dictionary_page_size_limit",)?,- statistics_enabled: proto.statistics_enabled_opt.as_ref().map(- |opt| match opt {+ statistics_enabled: proto+ .statistics_enabled_opt+ .as_ref()+ .map(|opt| match opt {parquet_options::StatisticsEnabledOpt::StatisticsEnabled(statistics,- ) => statistics.clone(),- },- ),+ ) => statistics.parse(),+ })+ .transpose()?,max_row_group_size: to_usize(proto.max_row_group_size, "max_row_group_size")?,max_in_list_size: to_usize(proto.max_in_list_size, "max_in_list_size")?,created_by: proto.created_by.clone(),@@ -561,3 +563,26 @@ impl TryFrom<&TableParquetOptionsProto> for TableParquetOptions {})}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn rejects_invalid_parquet_statistics() {+ let proto = ParquetOptionsProto {+ statistics_enabled_opt: Some(+ parquet_options::StatisticsEnabledOpt::StatisticsEnabled(+ "invalid".to_string(),+ ),+ ),+ ..Default::default()+ };++ let err = ParquetOptions::try_from(&proto).unwrap_err();+ assert!(+ err.to_string()+ .contains("Invalid parquet statistics setting: invalid")+ );+ }+}diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex e36e59bccb..b1dfc7163c 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -732,6 +732,25 @@ statement error DataFusion error: Error during planning: Duration has overflowedSET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'# Set invalid value and ensures error+statement ok+SET datafusion.execution.parquet.statistics_enabled = 'CHUNK'++query TT+SHOW datafusion.execution.parquet.statistics_enabled+----+datafusion.execution.parquet.statistics_enabled chunk++statement error+SET datafusion.execution.parquet.statistics_enabled = 'invalid'+----+DataFusion error: Error setting config datafusion.execution.parquet.statistics_enabled+caused by+Invalid or Unsupported Configuration: Invalid parquet statistics setting: invalid. Expected one of: none, chunk, page+++statement ok+RESET datafusion.execution.parquet.statistics_enabled+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0SET datafusion.execution.batch_size = 0