Apache DataFusion SQL Query Engine
Refactor PartitionedFile: add ordering field and new_from_meta constructor (#19596)
Adrian Garcia Badaraccoed01b67f2d7c
diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rsindex 6bbb63f6a1..b04238ebc9 100644--- a/datafusion/core/src/datasource/file_format/mod.rs+++ b/datafusion/core/src/datasource/file_format/mod.rs@@ -73,17 +73,7 @@ pub(crate) mod test_util {.infer_stats(state, &store, file_schema.clone(), &meta).await?;- let file_groups = vec![- vec![PartitionedFile {- object_meta: meta,- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- }]- .into(),- ];+ let file_groups = vec![vec![PartitionedFile::new_from_meta(meta)].into()];let exec = format.create_physical_plan(diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rsindex aefda64d39..1f21d6a7e6 100644--- a/datafusion/core/src/datasource/mod.rs+++ b/datafusion/core/src/datasource/mod.rs@@ -113,14 +113,7 @@ mod tests {version: None,};- let partitioned_file = PartitionedFile {- object_meta: meta,- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- };+ let partitioned_file = PartitionedFile::new_from_meta(meta);let f1 = Field::new("id", DataType::Int32, true);let f2 = Field::new("extra_column", DataType::Utf8, true);diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rsindex 4703b55ecc..ce2b05e6d3 100644--- a/datafusion/core/src/datasource/physical_plan/parquet.rs+++ b/datafusion/core/src/datasource/physical_plan/parquet.rs@@ -54,7 +54,7 @@ mod tests {use datafusion_datasource::source::DataSourceExec;use datafusion_datasource::file::FileSource;- use datafusion_datasource::{FileRange, PartitionedFile, TableSchema};+ use datafusion_datasource::{PartitionedFile, TableSchema};use datafusion_datasource_parquet::source::ParquetSource;use datafusion_datasource_parquet::{DefaultParquetFileReaderFactory, ParquetFileReaderFactory, ParquetFormat,@@ -1527,14 +1527,7 @@ mod tests {#[tokio::test]async fn parquet_exec_with_range() -> Result<()> {fn file_range(meta: &ObjectMeta, start: i64, end: i64) -> PartitionedFile {- PartitionedFile {- object_meta: meta.clone(),- partition_values: vec![],- range: Some(FileRange { start, end }),- statistics: None,- extensions: None,- metadata_size_hint: None,- }+ PartitionedFile::new_from_meta(meta.clone()).with_range(start, end)}async fn assert_parquet_read(@@ -1616,21 +1609,15 @@ mod tests {.await.unwrap();- let partitioned_file = PartitionedFile {- object_meta: meta,- partition_values: vec![+ let partitioned_file = PartitionedFile::new_from_meta(meta)+ .with_partition_values(vec![ScalarValue::from("2021"),ScalarValue::UInt8(Some(10)),ScalarValue::Dictionary(Box::new(DataType::UInt16),Box::new(ScalarValue::from("26")),),- ],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- };+ ]);let expected_schema = Schema::new(vec![Field::new("id", DataType::Int32, true),@@ -1711,20 +1698,13 @@ mod tests {.unwrap().child("invalid.parquet");- let partitioned_file = PartitionedFile {- object_meta: ObjectMeta {- location,- last_modified: Utc.timestamp_nanos(0),- size: 1337,- e_tag: None,- version: None,- },- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- };+ let partitioned_file = PartitionedFile::new_from_meta(ObjectMeta {+ location,+ last_modified: Utc.timestamp_nanos(0),+ size: 1337,+ e_tag: None,+ version: None,+ });let file_schema = Arc::new(Schema::empty());let config = FileScanConfigBuilder::new(@@ -2376,36 +2356,22 @@ mod tests {);let config = FileScanConfigBuilder::new(store_url, source).with_file(- PartitionedFile {- object_meta: ObjectMeta {- location: Path::from(name_1),- last_modified: Utc::now(),- size: total_size_1,- e_tag: None,- version: None,- },- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- }- .with_metadata_size_hint(123),- )- .with_file(PartitionedFile {- object_meta: ObjectMeta {- location: Path::from(name_2),+ PartitionedFile::new_from_meta(ObjectMeta {+ location: Path::from(name_1),last_modified: Utc::now(),- size: total_size_2,+ size: total_size_1,e_tag: None,version: None,- },- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- })+ })+ .with_metadata_size_hint(123),+ )+ .with_file(PartitionedFile::new_from_meta(ObjectMeta {+ location: Path::from(name_2),+ last_modified: Utc::now(),+ size: total_size_2,+ e_tag: None,+ version: None,+ })).build();let exec = DataSourceExec::from_data_source(config);diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rsindex 44e884c23a..53684e51bc 100644--- a/datafusion/core/src/test_util/parquet.rs+++ b/datafusion/core/src/test_util/parquet.rs@@ -160,16 +160,8 @@ impl TestParquetFile {.with_table_parquet_options(parquet_options.clone()),);let scan_config_builder =- FileScanConfigBuilder::new(self.object_store_url.clone(), source).with_file(- PartitionedFile {- object_meta: self.object_meta.clone(),- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- },- );+ FileScanConfigBuilder::new(self.object_store_url.clone(), source)+ .with_file(PartitionedFile::new_from_meta(self.object_meta.clone()));let df_schema = Arc::clone(&self.schema).to_dfschema_ref()?;diff --git a/datafusion/core/tests/parquet/custom_reader.rs b/datafusion/core/tests/parquet/custom_reader.rsindex 31ec6efd19..25f69d2975 100644--- a/datafusion/core/tests/parquet/custom_reader.rs+++ b/datafusion/core/tests/parquet/custom_reader.rs@@ -69,13 +69,9 @@ async fn route_data_access_ops_to_parquet_file_reader_factory() {store_parquet_in_memory(vec![batch]).await;let file_group = parquet_files_meta.into_iter()- .map(|meta| PartitionedFile {- object_meta: meta,- partition_values: vec![],- range: None,- statistics: None,- extensions: Some(Arc::new(String::from(EXPECTED_USER_DEFINED_METADATA))),- metadata_size_hint: None,+ .map(|meta| {+ PartitionedFile::new_from_meta(meta)+ .with_extensions(Arc::new(String::from(EXPECTED_USER_DEFINED_METADATA)))}).collect();diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rsindex 17392974b6..7eb39bfe78 100644--- a/datafusion/core/tests/parquet/page_pruning.rs+++ b/datafusion/core/tests/parquet/page_pruning.rs@@ -67,14 +67,7 @@ async fn get_parquet_exec(.await.unwrap();- let partitioned_file = PartitionedFile {- object_meta: meta,- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- };+ let partitioned_file = PartitionedFile::new_from_meta(meta);let df_schema = schema.clone().to_dfschema().unwrap();let execution_props = ExecutionProps::new();diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 1264197609..046379cc25 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -1559,14 +1559,7 @@ mod tests {ParquetObjectReader::new(Arc::new(in_memory), object_meta.location.clone()).with_file_size(object_meta.size);- let partitioned_file = PartitionedFile {- object_meta,- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- };+ let partitioned_file = PartitionedFile::new_from_meta(object_meta);let reader = ParquetFileReader {inner,diff --git a/datafusion/datasource/src/display.rs b/datafusion/datasource/src/display.rsindex 15fe8679ac..0f59e33ff9 100644--- a/datafusion/datasource/src/display.rs+++ b/datafusion/datasource/src/display.rs@@ -287,13 +287,6 @@ mod tests {version: None,};- PartitionedFile {- object_meta,- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- }+ PartitionedFile::new_from_meta(object_meta)}}diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex c8636343cc..1f7c37315c 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -1661,43 +1661,40 @@ mod tests {impl From<File> for PartitionedFile {fn from(file: File) -> Self {- PartitionedFile {- object_meta: ObjectMeta {- location: Path::from(format!(- "data/date={}/{}.parquet",- file.date, file.name- )),- last_modified: chrono::Utc.timestamp_nanos(0),- size: 0,- e_tag: None,- version: None,- },- partition_values: vec![ScalarValue::from(file.date)],- range: None,- statistics: Some(Arc::new(Statistics {- num_rows: Precision::Absent,- total_byte_size: Precision::Absent,- column_statistics: file- .statistics- .into_iter()- .map(|stats| {- stats- .map(|(min, max)| ColumnStatistics {- min_value: Precision::Exact(- ScalarValue::Float64(min),- ),- max_value: Precision::Exact(- ScalarValue::Float64(max),- ),- ..Default::default()- })- .unwrap_or_default()- })- .collect::<Vec<_>>(),- })),- extensions: None,- metadata_size_hint: None,- }+ let object_meta = ObjectMeta {+ location: Path::from(format!(+ "data/date={}/{}.parquet",+ file.date, file.name+ )),+ last_modified: chrono::Utc.timestamp_nanos(0),+ size: 0,+ e_tag: None,+ version: None,+ };+ let statistics = Arc::new(Statistics {+ num_rows: Precision::Absent,+ total_byte_size: Precision::Absent,+ column_statistics: file+ .statistics+ .into_iter()+ .map(|stats| {+ stats+ .map(|(min, max)| ColumnStatistics {+ min_value: Precision::Exact(ScalarValue::Float64(+ min,+ )),+ max_value: Precision::Exact(ScalarValue::Float64(+ max,+ )),+ ..Default::default()+ })+ .unwrap_or_default()+ })+ .collect::<Vec<_>>(),+ });+ PartitionedFile::new_from_meta(object_meta)+ .with_partition_values(vec![ScalarValue::from(file.date)])+ .with_statistics(statistics)}}}diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rsindex 744ec667d5..2965be7637 100644--- a/datafusion/datasource/src/mod.rs+++ b/datafusion/datasource/src/mod.rs@@ -58,6 +58,7 @@ use chrono::TimeZone;use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, Result, exec_datafusion_err};use datafusion_common::{ScalarValue, Statistics};+use datafusion_physical_expr::LexOrdering;use futures::{Stream, StreamExt};use object_store::{GetOptions, GetRange, ObjectStore};use object_store::{ObjectMeta, path::Path};@@ -133,6 +134,16 @@ pub struct PartitionedFile {/// When set via [`Self::with_statistics`], partition column statistics are automatically/// computed from [`Self::partition_values`] with exact min/max/null_count/distinct_count.pub statistics: Option<Arc<Statistics>>,+ /// The known lexicographical ordering of the rows in this file, if any.+ ///+ /// This describes how the data within the file is sorted with respect to one or more+ /// columns, and is used by the optimizer for planning operations that depend on input+ /// ordering (e.g. merges, sorts, and certain aggregations).+ ///+ /// When available, this is typically inferred from file-level metadata exposed by the+ /// underlying format (for example, Parquet `sorting_columns`), but it may also be set+ /// explicitly via [`Self::with_ordering`].+ pub ordering: Option<LexOrdering>,/// An optional field for user defined per object metadatapub extensions: Option<Arc<dyn std::any::Any + Send + Sync>>,/// The estimated size of the parquet metadata, in bytes@@ -153,6 +164,20 @@ impl PartitionedFile {partition_values: vec![],range: None,statistics: None,+ ordering: None,+ extensions: None,+ metadata_size_hint: None,+ }+ }++ /// Create a file from a known ObjectMeta without partition+ pub fn new_from_meta(object_meta: ObjectMeta) -> Self {+ Self {+ object_meta,+ partition_values: vec![],+ range: None,+ statistics: None,+ ordering: None,extensions: None,metadata_size_hint: None,}@@ -171,12 +196,20 @@ impl PartitionedFile {partition_values: vec![],range: Some(FileRange { start, end }),statistics: None,+ ordering: None,extensions: None,metadata_size_hint: None,}.with_range(start, end)}+ /// Attach partition values to this file.+ /// This replaces any existing partition values.+ pub fn with_partition_values(mut self, partition_values: Vec<ScalarValue>) -> Self {+ self.partition_values = partition_values;+ self+ }+/// Size of the file to be scanned (taking into account the range, if present).pub fn effective_size(&self) -> u64 {if let Some(range) = &self.range {@@ -282,6 +315,15 @@ impl PartitionedFile {false}}++ /// Set the known ordering of data in this file.+ ///+ /// The ordering represents the lexicographical sort order of the data,+ /// typically inferred from file metadata (e.g., Parquet sorting_columns).+ pub fn with_ordering(mut self, ordering: Option<LexOrdering>) -> Self {+ self.ordering = ordering;+ self+ }}impl From<ObjectMeta> for PartitionedFile {@@ -291,6 +333,7 @@ impl From<ObjectMeta> for PartitionedFile {partition_values: vec![],range: None,statistics: None,+ ordering: None,extensions: None,metadata_size_hint: None,}@@ -487,6 +530,7 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGrobyte_size: Precision::Absent,}],})),+ ordering: None,extensions: None,metadata_size_hint: None,};diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 073fdd858c..3cfc796700 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -616,30 +616,28 @@ impl TryFrom<&protobuf::PartitionedFile> for PartitionedFile {type Error = DataFusionError;fn try_from(val: &protobuf::PartitionedFile) -> Result<Self, Self::Error> {- Ok(PartitionedFile {- object_meta: ObjectMeta {- location: Path::parse(val.path.as_str()).map_err(|e| {- proto_error(format!("Invalid object_store path: {e}"))- })?,- last_modified: Utc.timestamp_nanos(val.last_modified_ns as i64),- size: val.size,- e_tag: None,- version: None,- },- partition_values: val- .partition_values+ let mut pf = PartitionedFile::new_from_meta(ObjectMeta {+ location: Path::parse(val.path.as_str())+ .map_err(|e| proto_error(format!("Invalid object_store path: {e}")))?,+ last_modified: Utc.timestamp_nanos(val.last_modified_ns as i64),+ size: val.size,+ e_tag: None,+ version: None,+ })+ .with_partition_values(+ val.partition_values.iter().map(|v| v.try_into()).collect::<Result<Vec<_>, _>>()?,- range: val.range.as_ref().map(|v| v.try_into()).transpose()?,- statistics: val- .statistics- .as_ref()- .map(|v| v.try_into().map(Arc::new))- .transpose()?,- extensions: None,- metadata_size_hint: None,- })+ );+ if let Some(range) = val.range.as_ref() {+ let file_range: FileRange = range.try_into()?;+ pf = pf.with_range(file_range.start, file_range.end);+ }+ if let Some(proto_stats) = val.statistics.as_ref() {+ pf = pf.with_statistics(Arc::new(proto_stats.try_into()?));+ }+ Ok(pf)}}@@ -754,20 +752,13 @@ mod tests {#[test]fn partitioned_file_path_roundtrip_percent_encoded() {let path_str = "foo/foo%2Fbar/baz%252Fqux";- let pf = PartitionedFile {- object_meta: ObjectMeta {- location: Path::parse(path_str).unwrap(),- last_modified: Utc.timestamp_nanos(1_000),- size: 42,- e_tag: None,- version: None,- },- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- };+ let pf = PartitionedFile::new_from_meta(ObjectMeta {+ location: Path::parse(path_str).unwrap(),+ last_modified: Utc.timestamp_nanos(1_000),+ size: 42,+ e_tag: None,+ version: None,+ });let proto = protobuf::PartitionedFile::try_from(&pf).unwrap();assert_eq!(proto.path, path_str);diff --git a/datafusion/substrait/src/physical_plan/consumer.rs b/datafusion/substrait/src/physical_plan/consumer.rsindex ac0f267225..ccaf1abec4 100644--- a/datafusion/substrait/src/physical_plan/consumer.rs+++ b/datafusion/substrait/src/physical_plan/consumer.rs@@ -119,20 +119,14 @@ pub async fn from_substrait_rel(.unwrap();let size = 0;- let partitioned_file = PartitionedFile {- object_meta: ObjectMeta {+ let partitioned_file =+ PartitionedFile::new_from_meta(ObjectMeta {last_modified: last_modified.into(),location: path.into(),size,e_tag: None,version: None,- },- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- };+ });let part_index = file.partition_index as usize;while part_index >= file_groups.len() {