Apache DataFusion SQL Query Engine
fix(proto): check integer conversions in common options and constraints (#24726)
Burak Şen54f5cf162c3b
diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rsindex f14924563c..a8c27369cb 100644--- a/datafusion/datasource-csv/src/file_format.rs+++ b/datafusion/datasource-csv/src/file_format.rs@@ -941,7 +941,7 @@ impl CsvSink {/// Encode a [`CsvFormatFactory`]'s options as their protobuf form.///-/// The reverse direction is `From<&protobuf::CsvOptions> for CsvOptions` in+/// The reverse direction is `TryFrom<&protobuf::CsvOptions> for CsvOptions` in/// `datafusion-proto-models`: `CsvOptions` is a `datafusion-common` type, so/// that half cannot live here.#[cfg(feature = "proto")]diff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rsindex 211011f95c..2a9900bc23 100644--- a/datafusion/datasource-json/src/file_format.rs+++ b/datafusion/datasource-json/src/file_format.rs@@ -618,7 +618,7 @@ impl Decoder for JsonDecoder {/// Encode a [`JsonFormatFactory`]'s options as their protobuf form.///-/// The reverse direction is `From<&protobuf::JsonOptions> for JsonOptions` in+/// The reverse direction is `TryFrom<&protobuf::JsonOptions> for JsonOptions` in/// `datafusion-proto-models`: `JsonOptions` is a `datafusion-common` type, so/// that half cannot live here.#[cfg(feature = "proto")]diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex cf3a3cb75a..7dab16406b 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -46,6 +46,7 @@ use datafusion_common::{parsers::CompressionTypeVariant,plan_datafusion_err,stats::Precision,+ utils::usize_from_wire,};#[derive(Debug)]@@ -738,30 +739,23 @@ impl From<protobuf::IntervalUnit> for IntervalUnit {}}-impl From<protobuf::Constraints> for Constraints {- fn from(constraints: protobuf::Constraints) -> Self {- Constraints::new_unverified(- constraints- .constraints- .into_iter()- .map(|item| item.into())- .collect(),- )+impl TryFrom<protobuf::Constraints> for Constraints {+ type Error = DataFusionError;++ fn try_from(+ constraints: protobuf::Constraints,+ ) -> datafusion_common::Result<Self, Self::Error> {+ Self::try_from(&constraints)}}-impl From<protobuf::Constraint> for Constraint {- fn from(value: protobuf::Constraint) -> Self {- match value.constraint_mode.unwrap() {- protobuf::constraint::ConstraintMode::PrimaryKey(elem) => {- Constraint::PrimaryKey(- elem.indices.into_iter().map(|item| item as usize).collect(),- )- }- protobuf::constraint::ConstraintMode::Unique(elem) => Constraint::Unique(- elem.indices.into_iter().map(|item| item as usize).collect(),- ),- }+impl TryFrom<protobuf::Constraint> for Constraint {+ type Error = DataFusionError;++ fn try_from(+ value: protobuf::Constraint,+ ) -> datafusion_common::Result<Self, Self::Error> {+ Constraint::try_from(&value)}}@@ -884,20 +878,28 @@ impl From<protobuf::JoinSide> for JoinSide {}}-impl From<&protobuf::Constraint> for Constraint {- fn from(value: &protobuf::Constraint) -> Self {+impl TryFrom<&protobuf::Constraint> for Constraint {+ type Error = DataFusionError;++ fn try_from(+ value: &protobuf::Constraint,+ ) -> datafusion_common::Result<Self, Self::Error> {+ let decode_indices = |indices: &[u64]| {+ indices+ .iter()+ .map(|&index| usize_from_wire(index, "Constraint", "indices"))+ .collect::<datafusion_common::Result<Vec<_>>>()+ };match &value.constraint_mode {Some(protobuf::constraint::ConstraintMode::PrimaryKey(elem)) => {- Constraint::PrimaryKey(- elem.indices.iter().map(|&item| item as usize).collect(),- )+ Ok(Constraint::PrimaryKey(decode_indices(&elem.indices)?))}Some(protobuf::constraint::ConstraintMode::Unique(elem)) => {- Constraint::Unique(- elem.indices.iter().map(|&item| item as usize).collect(),- )+ Ok(Constraint::Unique(decode_indices(&elem.indices)?))}- None => panic!("constraint_mode not set"),+ None => Err(plan_datafusion_err!(+ "Constraint: missing required field 'constraint_mode'"+ )),}}}@@ -912,8 +914,8 @@ impl TryFrom<&protobuf::Constraints> for Constraints {constraints.constraints.iter()- .map(|item| item.into())- .collect(),+ .map(Constraint::try_from)+ .collect::<datafusion_common::Result<Vec<_>>>()?,))}}@@ -1016,7 +1018,10 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions {newlines_in_values: proto_opts.newlines_in_values.first().map(|h| *h != 0),compression: proto_opts.compression().into(),compression_level: proto_opts.compression_level,- schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),+ schema_infer_max_rec: proto_opts+ .schema_infer_max_rec+ .map(|v| usize_from_wire(v, "CsvOptions", "schema_infer_max_rec"))+ .transpose()?,date_format: (!proto_opts.date_format.is_empty()).then(|| proto_opts.date_format.clone()),datetime_format: (!proto_opts.datetime_format.is_empty())@@ -1052,6 +1057,8 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {fn try_from(value: &protobuf::ParquetOptions,) -> datafusion_common::Result<Self, Self::Error> {+ let to_usize =+ |value: u64, field: &str| usize_from_wire(value, "ParquetOptions", field);Ok(ParquetOptions {enable_page_index: value.enable_page_index,pruning: value.pruning,@@ -1059,14 +1066,19 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {metadata_size_hint: value.metadata_size_hint_opt.map(|opt| match opt {- protobuf::parquet_options::MetadataSizeHintOpt::MetadataSizeHint(v) => Some(v as usize),+ protobuf::parquet_options::MetadataSizeHintOpt::MetadataSizeHint(v) => {+ to_usize(v, "metadata_size_hint")+ }})- .unwrap_or(None),+ .transpose()?,pushdown_filters: value.pushdown_filters,reorder_filters: value.reorder_filters,force_filter_selections: value.force_filter_selections,- data_pagesize_limit: value.data_pagesize_limit as usize,- write_batch_size: value.write_batch_size as usize,+ data_pagesize_limit: to_usize(+ value.data_pagesize_limit,+ "data_pagesize_limit",+ )?,+ write_batch_size: to_usize(value.write_batch_size, "write_batch_size")?,writer_version: value.writer_version.parse().map_err(|e| {DataFusionError::Internal(format!("Failed to parse writer_version: {e}"))})?,@@ -1075,29 +1087,39 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {}).unwrap_or(None),dictionary_enabled: value.dictionary_enabled_opt.as_ref().map(|protobuf::parquet_options::DictionaryEnabledOpt::DictionaryEnabled(v)| *v),// Continuing from where we left off in the TryFrom implementation- dictionary_page_size_limit: value.dictionary_page_size_limit as usize,+ dictionary_page_size_limit: to_usize(+ value.dictionary_page_size_limit,+ "dictionary_page_size_limit",+ )?,statistics_enabled: value.statistics_enabled_opt.clone().map(|opt| match opt {protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v) => Some(v),}).unwrap_or(None),- max_row_group_size: value.max_row_group_size as usize,- max_in_list_size: value.max_in_list_size as usize,+ max_row_group_size: to_usize(value.max_row_group_size, "max_row_group_size")?,+ max_in_list_size: to_usize(value.max_in_list_size, "max_in_list_size")?,created_by: value.created_by.clone(),column_index_truncate_length: value.column_index_truncate_length_opt.as_ref().map(|opt| match opt {- protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v) => Some(*v as usize),+ protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v) => {+ to_usize(*v, "column_index_truncate_length")+ }})- .unwrap_or(None),+ .transpose()?,statistics_truncate_length: value.statistics_truncate_length_opt.as_ref().map(|opt| match opt {- protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v) => Some(*v as usize),+ protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v) => {+ to_usize(*v, "statistics_truncate_length")+ }})- .unwrap_or(None),- data_page_row_count_limit: value.data_page_row_count_limit as usize,+ .transpose()?,+ data_page_row_count_limit: to_usize(+ value.data_page_row_count_limit,+ "data_page_row_count_limit",+ )?,encoding: value.encoding_opt.clone().map(|opt| match opt {@@ -1119,8 +1141,14 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {}).unwrap_or(None),allow_single_file_parallelism: value.allow_single_file_parallelism,- maximum_parallel_row_group_writers: value.maximum_parallel_row_group_writers as usize,- maximum_buffered_record_batches_per_stream: value.maximum_buffered_record_batches_per_stream as usize,+ maximum_parallel_row_group_writers: to_usize(+ value.maximum_parallel_row_group_writers,+ "maximum_parallel_row_group_writers",+ )?,+ maximum_buffered_record_batches_per_stream: to_usize(+ value.maximum_buffered_record_batches_per_stream,+ "maximum_buffered_record_batches_per_stream",+ )?,schema_force_view_types: value.schema_force_view_types,binary_as_string: value.binary_as_string,coerce_int96: value.coerce_int96_opt.clone().map(|opt| match opt {@@ -1131,24 +1159,34 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {}).unwrap_or(None),skip_arrow_metadata: value.skip_arrow_metadata,max_predicate_cache_size: value.max_predicate_cache_size_opt.map(|opt| match opt {- protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v) => Some(v as usize),- }).unwrap_or(None),- max_row_group_bytes: value.max_row_group_bytes_opt.and_then(|opt| match opt {- protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v) => MaxRowGroupBytes::try_new(v as usize).ok(),- }),- content_defined_chunking: value.content_defined_chunking.map(ParquetCdcOptions::from).unwrap_or_default(),+ protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v) => {+ to_usize(v, "max_predicate_cache_size")+ }+ }).transpose()?,+ max_row_group_bytes: value.max_row_group_bytes_opt.map(|opt| match opt {+ protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v) => {+ MaxRowGroupBytes::try_new(to_usize(v, "max_row_group_bytes")?)+ }+ }).transpose()?,+ content_defined_chunking: value.content_defined_chunking.map(ParquetCdcOptions::try_from).transpose()?.unwrap_or_default(),})}}-impl From<protobuf::ParquetCdcOptions> for ParquetCdcOptions {- fn from(value: protobuf::ParquetCdcOptions) -> Self {- ParquetCdcOptions {+impl TryFrom<protobuf::ParquetCdcOptions> for ParquetCdcOptions {+ type Error = DataFusionError;++ fn try_from(+ value: protobuf::ParquetCdcOptions,+ ) -> datafusion_common::Result<Self, Self::Error> {+ let to_usize =+ |value: u64, field: &str| usize_from_wire(value, "ParquetCdcOptions", field);+ Ok(ParquetCdcOptions {enabled: value.enabled,- min_chunk_size: value.min_chunk_size as usize,- max_chunk_size: value.max_chunk_size as usize,+ min_chunk_size: to_usize(value.min_chunk_size, "min_chunk_size")?,+ max_chunk_size: to_usize(value.max_chunk_size, "max_chunk_size")?,norm_level: value.norm_level,- }+ })}}@@ -1214,9 +1252,9 @@ impl TryFrom<&protobuf::TableParquetOptions> for TableParquetOptions {global: value.global.as_ref()- .map(|v| v.try_into())- .unwrap()- .unwrap(),+ .map(ParquetOptions::try_from)+ .transpose()?+ .unwrap_or_default(),column_specific_options,..Default::default()};@@ -1234,7 +1272,10 @@ impl TryFrom<&protobuf::JsonOptions> for JsonOptions {Ok(JsonOptions {compression: compression.into(),compression_level: proto_opts.compression_level,- schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),+ schema_infer_max_rec: proto_opts+ .schema_infer_max_rec+ .map(|v| usize_from_wire(v, "JsonOptions", "schema_infer_max_rec"))+ .transpose()?,newline_delimited: proto_opts.newline_delimited.unwrap_or(true),})}@@ -1338,6 +1379,72 @@ mod tests {MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,};+ #[test]+ fn constraint_requires_mode() {+ let err = datafusion_common::Constraint::try_from(+ &crate::protobuf_common::Constraint {+ constraint_mode: None,+ },+ )+ .unwrap_err();+ assert!(+ err.to_string()+ .contains("Constraint: missing required field 'constraint_mode'")+ );+ }++ #[test]+ fn table_parquet_options_defaults_missing_global() {+ let recovered = TableParquetOptions::try_from(+ &crate::protobuf_common::TableParquetOptions::default(),+ )+ .expect("missing global should use defaults");++ assert_eq!(recovered, TableParquetOptions::default());+ }++ #[test]+ fn table_parquet_options_checks_nested_usize_conversion() {+ let proto = crate::protobuf_common::TableParquetOptions {+ global: Some(crate::protobuf_common::ParquetOptions {+ data_pagesize_limit: u64::MAX,+ writer_version: "1.0".to_string(),+ ..Default::default()+ }),+ ..Default::default()+ };+ let recovered = TableParquetOptions::try_from(&proto);++ if usize::BITS >= u64::BITS {+ assert_eq!(recovered.unwrap().global.data_pagesize_limit, usize::MAX);+ } else {+ assert_eq!(+ recovered.unwrap_err().strip_backtrace(),+ "Error during planning: ParquetOptions: data_pagesize_limit wire value 18446744073709551615 is out of range for usize"+ );+ }+ }++ #[test]+ fn table_parquet_options_propagates_nested_conversion_error() {+ let proto = crate::protobuf_common::TableParquetOptions {+ global: Some(crate::protobuf_common::ParquetOptions {+ writer_version: "1.0".to_string(),+ max_row_group_bytes_opt: Some(+ crate::protobuf_common::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(0),+ ),+ ..Default::default()+ }),+ ..Default::default()+ };++ let err = TableParquetOptions::try_from(&proto).unwrap_err();+ assert_eq!(+ err.strip_backtrace(),+ "Invalid or Unsupported Configuration: max_row_group_bytes must be greater than 0"+ );+ }+fn parquet_options_proto_round_trip(opts: ParquetOptions) -> ParquetOptions {let proto: crate::protobuf_common::ParquetOptions =(&opts).try_into().expect("to_proto");diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rsindex 74ead8c520..16903ec388 100644--- a/datafusion/proto-models/src/from_proto.rs+++ b/datafusion/proto-models/src/from_proto.rs@@ -31,6 +31,7 @@ use datafusion_common::config::{};use datafusion_common::display::{PlanType, StringifiedPlan};use datafusion_common::parsers::{CompressionTypeVariant, CsvQuoteStyle};+use datafusion_common::utils::usize_from_wire;use datafusion_common::{JoinConstraint, JoinType, NullEquality, RecursionUnnestOption, TableReference,UnnestOptions,@@ -188,9 +189,11 @@ impl From<protobuf::NullEquality> for NullEquality {}}-impl From<&CsvOptionsProto> for CsvOptions {- fn from(proto: &CsvOptionsProto) -> Self {- CsvOptions {+impl TryFrom<&CsvOptionsProto> for CsvOptions {+ type Error = datafusion_common::DataFusionError;++ fn try_from(proto: &CsvOptionsProto) -> datafusion_common::Result<Self, Self::Error> {+ Ok(CsvOptions {has_header: if !proto.has_header.is_empty() {Some(proto.has_header[0] != 0)} else {@@ -220,7 +223,10 @@ impl From<&CsvOptionsProto> for CsvOptions {3 => CompressionTypeVariant::ZSTD,_ => CompressionTypeVariant::UNCOMPRESSED,},- schema_infer_max_rec: proto.schema_infer_max_rec.map(|v| v as usize),+ schema_infer_max_rec: proto+ .schema_infer_max_rec+ .map(|value| usize_from_wire(value, "CsvOptions", "schema_infer_max_rec"))+ .transpose()?,date_format: if proto.date_format.is_empty() {None} else {@@ -289,13 +295,17 @@ impl From<&CsvOptionsProto> for CsvOptions {} else {Some(proto.ignore_trailing_whitespace[0] != 0)},- }+ })}}-impl From<&JsonOptionsProto> for JsonOptions {- fn from(proto: &JsonOptionsProto) -> Self {- JsonOptions {+impl TryFrom<&JsonOptionsProto> for JsonOptions {+ type Error = datafusion_common::DataFusionError;++ fn try_from(+ proto: &JsonOptionsProto,+ ) -> datafusion_common::Result<Self, Self::Error> {+ Ok(JsonOptions {compression: match proto.compression {0 => CompressionTypeVariant::GZIP,1 => CompressionTypeVariant::BZIP2,@@ -303,21 +313,32 @@ impl From<&JsonOptionsProto> for JsonOptions {3 => CompressionTypeVariant::ZSTD,_ => CompressionTypeVariant::UNCOMPRESSED,},- schema_infer_max_rec: proto.schema_infer_max_rec.map(|v| v as usize),+ schema_infer_max_rec: proto+ .schema_infer_max_rec+ .map(|value| {+ usize_from_wire(value, "JsonOptions", "schema_infer_max_rec")+ })+ .transpose()?,compression_level: proto.compression_level,newline_delimited: proto.newline_delimited.unwrap_or(true),- }+ })}}-impl From<ParquetCdcOptionsProto> for ParquetCdcOptions {- fn from(value: ParquetCdcOptionsProto) -> Self {- ParquetCdcOptions {+impl TryFrom<ParquetCdcOptionsProto> for ParquetCdcOptions {+ type Error = datafusion_common::DataFusionError;++ fn try_from(+ value: ParquetCdcOptionsProto,+ ) -> datafusion_common::Result<Self, Self::Error> {+ let to_usize =+ |value: u64, field: &str| usize_from_wire(value, "ParquetCdcOptions", field);+ Ok(ParquetCdcOptions {enabled: value.enabled,- min_chunk_size: value.min_chunk_size as usize,- max_chunk_size: value.max_chunk_size as usize,+ min_chunk_size: to_usize(value.min_chunk_size, "min_chunk_size")?,+ max_chunk_size: to_usize(value.max_chunk_size, "max_chunk_size")?,norm_level: value.norm_level,- }+ })}}@@ -334,6 +355,8 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {"" => ParquetOptions::default().writer_version,version => version.parse()?,};+ let to_usize =+ |value: u64, field: &str| usize_from_wire(value, "ParquetOptions", field);Ok(ParquetOptions {enable_page_index: proto.enable_page_index,@@ -344,14 +367,18 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {.as_ref().map(|opt| match opt {parquet_options::MetadataSizeHintOpt::MetadataSizeHint(size) => {- *size as usize+ to_usize(*size, "metadata_size_hint")}- }),+ })+ .transpose()?,pushdown_filters: proto.pushdown_filters,reorder_filters: proto.reorder_filters,force_filter_selections: proto.force_filter_selections,- data_pagesize_limit: proto.data_pagesize_limit as usize,- write_batch_size: proto.write_batch_size as usize,+ data_pagesize_limit: to_usize(+ proto.data_pagesize_limit,+ "data_pagesize_limit",+ )?,+ write_batch_size: to_usize(proto.write_batch_size, "write_batch_size")?,writer_version,compression: proto.compression_opt.as_ref().map(|opt| match opt {parquet_options::CompressionOpt::Compression(compression) => {@@ -365,7 +392,10 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {) => *enabled,}}),- dictionary_page_size_limit: proto.dictionary_page_size_limit as usize,+ dictionary_page_size_limit: to_usize(+ proto.dictionary_page_size_limit,+ "dictionary_page_size_limit",+ )?,statistics_enabled: proto.statistics_enabled_opt.as_ref().map(|opt| match opt {parquet_options::StatisticsEnabledOpt::StatisticsEnabled(@@ -373,22 +403,31 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {) => statistics.clone(),},),- max_row_group_size: proto.max_row_group_size as usize,- max_in_list_size: proto.max_in_list_size as usize,+ max_row_group_size: to_usize(proto.max_row_group_size, "max_row_group_size")?,+ max_in_list_size: to_usize(proto.max_in_list_size, "max_in_list_size")?,created_by: proto.created_by.clone(),column_index_truncate_length: proto.column_index_truncate_length_opt.as_ref().map(|opt| match opt {- parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(length) => *length as usize,- }),+ parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(length) => {+ to_usize(*length, "column_index_truncate_length")+ }+ })+ .transpose()?,statistics_truncate_length: proto.statistics_truncate_length_opt.as_ref().map(|opt| match opt {- parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(length) => *length as usize,- }),- data_page_row_count_limit: proto.data_page_row_count_limit as usize,+ parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(length) => {+ to_usize(*length, "statistics_truncate_length")+ }+ })+ .transpose()?,+ data_page_row_count_limit: to_usize(+ proto.data_page_row_count_limit,+ "data_page_row_count_limit",+ )?,encoding: proto.encoding_opt.as_ref().map(|opt| match opt {parquet_options::EncodingOpt::Encoding(encoding) => {encoding.clone()@@ -409,12 +448,14 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {parquet_options::BloomFilterNdvOpt::BloomFilterNdv(ndv) => *ndv,}),allow_single_file_parallelism: proto.allow_single_file_parallelism,- maximum_parallel_row_group_writers: proto- .maximum_parallel_row_group_writers- as usize,- maximum_buffered_record_batches_per_stream: proto- .maximum_buffered_record_batches_per_stream- as usize,+ maximum_parallel_row_group_writers: to_usize(+ proto.maximum_parallel_row_group_writers,+ "maximum_parallel_row_group_writers",+ )?,+ maximum_buffered_record_batches_per_stream: to_usize(+ proto.maximum_buffered_record_batches_per_stream,+ "maximum_buffered_record_batches_per_stream",+ )?,schema_force_view_types: proto.schema_force_view_types,binary_as_string: proto.binary_as_string,skip_arrow_metadata: proto.skip_arrow_metadata,@@ -437,19 +478,22 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {.map(|opt| match opt {parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(size,- ) => *size as usize,- }),+ ) => to_usize(*size, "max_predicate_cache_size"),+ })+ .transpose()?,max_row_group_bytes: proto.max_row_group_bytes_opt.as_ref()- .and_then(|opt| match opt {+ .map(|opt| match opt {parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(size) => {- MaxRowGroupBytes::try_new(*size as usize).ok()+ MaxRowGroupBytes::try_new(to_usize(*size, "max_row_group_bytes")?)}- }),+ })+ .transpose()?,content_defined_chunking: proto.content_defined_chunking- .map(ParquetCdcOptions::from)+ .map(ParquetCdcOptions::try_from)+ .transpose()?.unwrap_or_default(),})}diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 10c54cf55c..7c7a8ef639 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -77,7 +77,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {let proto = CsvOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode CsvOptionsProto: {e:?}")})?;- let options = CsvOptions::from(&proto);+ let options = CsvOptions::try_from(&proto)?;Ok(Arc::new(CsvFormatFactory {options: Some(options),}))@@ -155,7 +155,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {let proto = JsonOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode JsonOptionsProto: {e:?}")})?;- let options = JsonOptions::from(&proto);+ let options = JsonOptions::try_from(&proto)?;Ok(Arc::new(JsonFormatFactory {options: Some(options),}))diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 90d48b056c..647bceeea1 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -862,7 +862,7 @@ impl AsLogicalPlan for LogicalPlanNode {.with_definition(definition).with_unbounded(create_extern_table.unbounded).with_options(create_extern_table.options.clone())- .with_constraints(constraints.into())+ .with_constraints(constraints.try_into()?).with_column_defaults(column_defaults).build(),),diff --git a/datafusion/proto/tests/cases/public_conversions.rs b/datafusion/proto/tests/cases/public_conversions.rsindex 1cda8e01a0..ec71960f64 100644--- a/datafusion/proto/tests/cases/public_conversions.rs+++ b/datafusion/proto/tests/cases/public_conversions.rs@@ -35,7 +35,8 @@ use datafusion_common::config::{};use datafusion_common::display::StringifiedPlan;use datafusion_common::{- JoinConstraint, JoinType, NullEquality, TableReference, UnnestOptions,+ Constraint, Constraints, DataFusionError, JoinConstraint, JoinType, NullEquality,+ TableReference, UnnestOptions,};use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_sink_config::FileSinkConfig;@@ -48,6 +49,7 @@ use datafusion_expr::expr::NullTreatment;use datafusion_expr::{WindowFrame, WindowFrameBound, WindowFrameUnits};use datafusion_physical_expr::expressions::Column;use datafusion_proto::protobuf;+use datafusion_proto_common::protobuf_common;/// Asserts `T: From<F>` by naming the conversion.fn assert_from<F, T: From<F>>() {@@ -59,6 +61,12 @@ fn assert_try_from<F, T: TryFrom<F>>() {let _: fn(F) -> Result<T, T::Error> = TryFrom::try_from;}+/// Asserts `T: TryFrom<F, Error = DataFusionError>` rather than accepting the+/// blanket `TryFrom` implementation provided for an infallible `From`.+fn assert_datafusion_try_from<F, T: TryFrom<F, Error = DataFusionError>>() {+ let _: fn(F) -> Result<T, DataFusionError> = TryFrom::try_from;+}+#[test]fn file_scan_conversions_are_std_traits() {assert_try_from::<&protobuf::PartitionedFile, PartitionedFile>();@@ -112,16 +120,20 @@ fn common_type_conversions_are_std_traits() {assert_from::<JoinConstraint, protobuf::JoinConstraint>();assert_from::<protobuf::NullEquality, NullEquality>();assert_from::<NullEquality, protobuf::NullEquality>();+ assert_datafusion_try_from::<protobuf_common::Constraint, Constraint>();+ assert_datafusion_try_from::<&protobuf_common::Constraint, Constraint>();+ assert_datafusion_try_from::<protobuf_common::Constraints, Constraints>();+ assert_datafusion_try_from::<&protobuf_common::Constraints, Constraints>();}#[test]fn file_format_option_conversions_are_std_traits() {- assert_from::<&protobuf::CsvOptions, CsvOptions>();- assert_from::<&protobuf::JsonOptions, JsonOptions>();- assert_try_from::<&protobuf::ParquetOptions, ParquetOptions>();+ assert_datafusion_try_from::<&protobuf::CsvOptions, CsvOptions>();+ assert_datafusion_try_from::<&protobuf::JsonOptions, JsonOptions>();+ assert_datafusion_try_from::<&protobuf::ParquetOptions, ParquetOptions>();assert_from::<protobuf::ParquetColumnOptions, ParquetColumnOptions>();- assert_from::<protobuf::ParquetCdcOptions, ParquetCdcOptions>();- assert_try_from::<&protobuf::TableParquetOptions, TableParquetOptions>();+ assert_datafusion_try_from::<protobuf::ParquetCdcOptions, ParquetCdcOptions>();+ assert_datafusion_try_from::<&protobuf::TableParquetOptions, TableParquetOptions>();assert_from::<&CsvFormatFactory, protobuf::CsvOptions>();assert_from::<&JsonFormatFactory, protobuf::JsonOptions>();assert_from::<&ParquetFormatFactory, protobuf::TableParquetOptions>();diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 881aee0c63..0acde5640e 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -38,6 +38,31 @@ Implement `values_preserving`, call`selection.validate_num_groups(self.len())?` before reading, and return rows in`selection.iter()` order without changing the builder state.+### `datafusion-proto`: common option and constraint conversions are fallible++Protobuf conversions for `CsvOptions`, `JsonOptions`, `ParquetCdcOptions`,+`Constraint`, and `Constraints` now reject integer values that do not fit+`usize`. Their infallible `From` implementations have been replaced with+`TryFrom`. The existing `TryFrom` conversions for `ParquetOptions` and+`TableParquetOptions` now also validate all `usize`-backed fields. A constraint+without a `constraint_mode` returns an error instead of panicking.++**Migration guide:**++```rust,ignore+// Before+let csv = CsvOptions::from(&proto_csv);+let cdc = ParquetCdcOptions::from(proto_cdc);+let constraints: Constraints = proto_constraints.into();++// After+let csv = CsvOptions::try_from(&proto_csv)?;+let cdc = ParquetCdcOptions::try_from(proto_cdc)?;+let constraints = Constraints::try_from(proto_constraints)?;+```++See [issue #24170](https://github.com/apache/datafusion/issues/24170) for details.+### `datafusion.optimizer.use_statistics_registry` is deprecated and ignoredThe `datafusion.optimizer.use_statistics_registry` config flag is deprecated and