Apache DataFusion SQL Query Engine
feat(parquet): Implement `scan_efficiency_ratio` metric for parquet reading (#18577)
Peter Nguyenb0015d3a9337
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex cbc66581e9..71e9e840d9 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -887,6 +887,7 @@ async fn parquet_explain_analyze() {&formatted,"row_groups_pruned_statistics=1 total \u{2192} 1 matched");+ assert_contains!(&formatted, "scan_efficiency_ratio=14% (259/1851)");// The order of metrics is expected to be the same as the actual pruning order// (file-> row-group -> page)diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rsindex 306bc9e6b0..5eaa137e9a 100644--- a/datafusion/datasource-parquet/src/metrics.rs+++ b/datafusion/datasource-parquet/src/metrics.rs@@ -16,7 +16,8 @@// under the License.use datafusion_physical_plan::metrics::{- Count, ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics, Time,+ Count, ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics,+ RatioMergeStrategy, RatioMetrics, Time,};/// Stores metrics about the parquet execution for a particular parquet file.@@ -66,6 +67,8 @@ pub struct ParquetFileMetrics {pub page_index_eval_time: Time,/// Total time spent reading and parsing metadata from the footerpub metadata_load_time: Time,+ /// Scan Efficiency Ratio, calculated as bytes_scanned / total_file_size+ pub scan_efficiency_ratio: RatioMetrics,/// Predicate Cache: number of records read directly from the inner reader./// This is the number of rows decoded while evaluating predicatespub predicate_cache_inner_records: Count,@@ -114,6 +117,15 @@ impl ParquetFileMetrics {.with_type(MetricType::SUMMARY).pruning_metrics("files_ranges_pruned_statistics", partition);+ let scan_efficiency_ratio = MetricBuilder::new(metrics)+ .with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY)+ .ratio_metrics_with_strategy(+ "scan_efficiency_ratio",+ partition,+ RatioMergeStrategy::AddPartSetTotal,+ );+// -----------------------// 'dev' level metrics// -----------------------@@ -164,6 +176,7 @@ impl ParquetFileMetrics {bloom_filter_eval_time,page_index_eval_time,metadata_load_time,+ scan_efficiency_ratio,predicate_cache_inner_records,predicate_cache_records,}diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rsindex 6670109cf7..59a5da7b9d 100644--- a/datafusion/datasource-parquet/src/reader.rs+++ b/datafusion/datasource-parquet/src/reader.rs@@ -97,6 +97,7 @@ impl DefaultParquetFileReaderFactory {pub struct ParquetFileReader {pub file_metrics: ParquetFileMetrics,pub inner: ParquetObjectReader,+ pub partitioned_file: PartitionedFile,}impl AsyncFileReader for ParquetFileReader {@@ -129,6 +130,18 @@ impl AsyncFileReader for ParquetFileReader {}}+impl Drop for ParquetFileReader {+ fn drop(&mut self) {+ self.file_metrics+ .scan_efficiency_ratio+ .add_part(self.file_metrics.bytes_scanned.value());+ // Multiple ParquetFileReaders may run, so we set_total to avoid adding the total multiple times+ self.file_metrics+ .scan_efficiency_ratio+ .set_total(self.partitioned_file.object_meta.size as usize);+ }+}+impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory {fn create_reader(&self,@@ -156,6 +169,7 @@ impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory {Ok(Box::new(ParquetFileReader {inner,file_metrics,+ partitioned_file,}))}}@@ -306,6 +320,18 @@ impl AsyncFileReader for CachedParquetFileReader {}}+impl Drop for CachedParquetFileReader {+ fn drop(&mut self) {+ self.file_metrics+ .scan_efficiency_ratio+ .add_part(self.file_metrics.bytes_scanned.value());+ // Multiple ParquetFileReaders may run, so we set_total to avoid adding the total multiple times+ self.file_metrics+ .scan_efficiency_ratio+ .set_total(self.partitioned_file.object_meta.size as usize);+ }+}+/// Wrapper to implement [`FileMetadata`] for [`ParquetMetaData`].pub struct CachedParquetMetaData(Arc<ParquetMetaData>);diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 2043f75070..90e4e10d5a 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -1533,6 +1533,7 @@ mod tests {data: bytes::Bytes,pruning_predicate: &PruningPredicate,) -> Result<RowGroupAccessPlanFilter> {+ use datafusion_datasource::PartitionedFile;use object_store::{ObjectMeta, ObjectStore};let object_meta = ObjectMeta {@@ -1551,12 +1552,23 @@ mod tests {let metrics = ExecutionPlanMetricsSet::new();let file_metrics =ParquetFileMetrics::new(0, object_meta.location.as_ref(), &metrics);- let inner = ParquetObjectReader::new(Arc::new(in_memory), object_meta.location)- .with_file_size(object_meta.size);+ let inner =+ ParquetObjectReader::new(Arc::new(in_memory), object_meta.location.clone())+ .with_file_size(object_meta.size);++ let partitioned_file = PartitionedFile {+ object_meta,+ partition_values: vec![],+ range: None,+ statistics: None,+ extensions: None,+ metadata_size_hint: None,+ };let reader = ParquetFileReader {inner,file_metrics: file_metrics.clone(),+ partitioned_file,};let mut builder = ParquetRecordBatchStreamBuilder::new(reader).await.unwrap();diff --git a/datafusion/physical-plan/src/metrics/builder.rs b/datafusion/physical-plan/src/metrics/builder.rsindex 91b2440122..21f6762e73 100644--- a/datafusion/physical-plan/src/metrics/builder.rs+++ b/datafusion/physical-plan/src/metrics/builder.rs@@ -20,7 +20,7 @@use std::{borrow::Cow, sync::Arc};use crate::metrics::{- value::{PruningMetrics, RatioMetrics},+ value::{PruningMetrics, RatioMergeStrategy, RatioMetrics},MetricType,};@@ -283,7 +283,17 @@ impl<'a> MetricBuilder<'a> {name: impl Into<Cow<'static, str>>,partition: usize,) -> RatioMetrics {- let ratio_metrics = RatioMetrics::new();+ self.ratio_metrics_with_strategy(name, partition, RatioMergeStrategy::default())+ }++ /// Consumes self and creates a new [`RatioMetrics`] with a specific merge strategy+ pub fn ratio_metrics_with_strategy(+ self,+ name: impl Into<Cow<'static, str>>,+ partition: usize,+ merge_strategy: RatioMergeStrategy,+ ) -> RatioMetrics {+ let ratio_metrics = RatioMetrics::new().with_merge_strategy(merge_strategy);self.with_partition(partition).build(MetricValue::Ratio {name: name.into(),ratio_metrics: ratio_metrics.clone(),diff --git a/datafusion/physical-plan/src/metrics/mod.rs b/datafusion/physical-plan/src/metrics/mod.rsindex 613c031808..c39779090b 100644--- a/datafusion/physical-plan/src/metrics/mod.rs+++ b/datafusion/physical-plan/src/metrics/mod.rs@@ -36,8 +36,8 @@ pub use baseline::{BaselineMetrics, RecordOutput, SpillMetrics, SplitMetrics};pub use builder::MetricBuilder;pub use custom::CustomMetricValue;pub use value::{- Count, Gauge, MetricValue, PruningMetrics, RatioMetrics, ScopedTimerGuard, Time,- Timestamp,+ Count, Gauge, MetricValue, PruningMetrics, RatioMergeStrategy, RatioMetrics,+ ScopedTimerGuard, Time, Timestamp,};/// Something that tracks a value of interest (metric) of a DataFusiondiff --git a/datafusion/physical-plan/src/metrics/value.rs b/datafusion/physical-plan/src/metrics/value.rsindex 7f31f75794..a5032ec8de 100644--- a/datafusion/physical-plan/src/metrics/value.rs+++ b/datafusion/physical-plan/src/metrics/value.rs@@ -437,6 +437,15 @@ impl PruningMetrics {pub struct RatioMetrics {part: Arc<AtomicUsize>,total: Arc<AtomicUsize>,+ merge_strategy: RatioMergeStrategy,+}++#[derive(Debug, Clone, Default)]+pub enum RatioMergeStrategy {+ #[default]+ AddPartAddTotal,+ AddPartSetTotal,+ SetPartAddTotal,}impl RatioMetrics {@@ -445,9 +454,15 @@ impl RatioMetrics {Self {part: Arc::new(AtomicUsize::new(0)),total: Arc::new(AtomicUsize::new(0)),+ merge_strategy: RatioMergeStrategy::AddPartAddTotal,}}+ pub fn with_merge_strategy(mut self, merge_strategy: RatioMergeStrategy) -> Self {+ self.merge_strategy = merge_strategy;+ self+ }+/// Add `n` to the numerator (`part`) valuepub fn add_part(&self, n: usize) {self.part.fetch_add(n, Ordering::Relaxed);@@ -458,10 +473,32 @@ impl RatioMetrics {self.total.fetch_add(n, Ordering::Relaxed);}+ /// Set the numerator (`part`) value to `n`, overwriting any existing value+ pub fn set_part(&self, n: usize) {+ self.part.store(n, Ordering::Relaxed);+ }++ /// Set the denominator (`total`) value to `n`, overwriting any existing value+ pub fn set_total(&self, n: usize) {+ self.total.store(n, Ordering::Relaxed);+ }+/// Merge the value from `other` into `self`pub fn merge(&self, other: &Self) {- self.add_part(other.part());- self.add_total(other.total());+ match self.merge_strategy {+ RatioMergeStrategy::AddPartAddTotal => {+ self.add_part(other.part());+ self.add_total(other.total());+ }+ RatioMergeStrategy::AddPartSetTotal => {+ self.add_part(other.part());+ self.set_total(other.total());+ }+ RatioMergeStrategy::SetPartAddTotal => {+ self.set_part(other.part());+ self.add_total(other.total());+ }+ }}/// Return the numerator (`part`) value@@ -784,10 +821,17 @@ impl MetricValue {name: name.clone(),pruning_metrics: PruningMetrics::new(),},- Self::Ratio { name, .. } => Self::Ratio {- name: name.clone(),- ratio_metrics: RatioMetrics::new(),- },+ Self::Ratio {+ name,+ ratio_metrics,+ } => {+ let merge_strategy = ratio_metrics.merge_strategy.clone();+ Self::Ratio {+ name: name.clone(),+ ratio_metrics: RatioMetrics::new()+ .with_merge_strategy(merge_strategy),+ }+ }Self::Custom { name, value } => Self::Custom {name: name.clone(),value: value.new_empty(),@@ -1140,6 +1184,67 @@ mod tests {assert_eq!("0.033% (1/3000)", tiny_ratio.to_string());}+ #[test]+ fn test_ratio_set_methods() {+ let ratio_metrics = RatioMetrics::new();++ // Ensure set methods don't increment+ ratio_metrics.set_part(10);+ ratio_metrics.set_part(10);+ ratio_metrics.set_total(40);+ ratio_metrics.set_total(40);+ assert_eq!("25% (10/40)", ratio_metrics.to_string());++ let ratio_metrics = RatioMetrics::new();++ // Calling set should change the value+ ratio_metrics.set_part(10);+ ratio_metrics.set_part(30);+ ratio_metrics.set_total(40);+ ratio_metrics.set_total(50);+ assert_eq!("60% (30/50)", ratio_metrics.to_string());+ }++ #[test]+ fn test_ratio_merge_strategy() {+ // Test AddPartSetTotal strategy+ let ratio_metrics1 =+ RatioMetrics::new().with_merge_strategy(RatioMergeStrategy::AddPartSetTotal);++ ratio_metrics1.set_part(10);+ ratio_metrics1.set_total(40);+ assert_eq!("25% (10/40)", ratio_metrics1.to_string());+ let ratio_metrics2 =+ RatioMetrics::new().with_merge_strategy(RatioMergeStrategy::AddPartSetTotal);+ ratio_metrics2.set_part(20);+ ratio_metrics2.set_total(40);+ assert_eq!("50% (20/40)", ratio_metrics2.to_string());++ ratio_metrics1.merge(&ratio_metrics2);+ assert_eq!("75% (30/40)", ratio_metrics1.to_string());++ // Test SetPartAddTotal strategy+ let ratio_metrics1 =+ RatioMetrics::new().with_merge_strategy(RatioMergeStrategy::SetPartAddTotal);+ ratio_metrics1.set_part(20);+ ratio_metrics1.set_total(50);+ let ratio_metrics2 = RatioMetrics::new();+ ratio_metrics2.set_part(20);+ ratio_metrics2.set_total(50);+ ratio_metrics1.merge(&ratio_metrics2);+ assert_eq!("20% (20/100)", ratio_metrics1.to_string());++ // Test AddPartAddTotal strategy (default)+ let ratio_metrics1 = RatioMetrics::new();+ ratio_metrics1.set_part(20);+ ratio_metrics1.set_total(50);+ let ratio_metrics2 = RatioMetrics::new();+ ratio_metrics2.set_part(20);+ ratio_metrics2.set_total(50);+ ratio_metrics1.merge(&ratio_metrics2);+ assert_eq!("40% (40/100)", ratio_metrics1.to_string());+ }+#[test]fn test_display_timestamp() {let timestamp = Timestamp::new();