Apache DataFusion SQL Query Engine
fix: count Parquet decoding in the scan's elapsed_compute (#25486)
akashjainn0e3c398ee149
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex 2ca04ad6b3..67614cfbde 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -908,6 +908,64 @@ async fn parquet_explain_analyze() {);}+/// The Parquet scan's `elapsed_compute` must cover decoding, which dominates a+/// full scan, and not only the projection of batches that are already decoded.+/// See <https://github.com/apache/datafusion/issues/18195>.+#[tokio::test]+async fn parquet_scan_elapsed_compute_includes_decoding() -> Result<()> {+ use datafusion::datasource::source::DataSourceExec;+ use datafusion_common::instant::Instant;+ use std::time::Duration;++ // A single partition keeps the scan on one thread, so its compute time+ // cannot exceed the wall time of the query.+ let ctx =+ SessionContext::new_with_config(SessionConfig::new().with_target_partitions(1));+ let tmp_dir = TempDir::new()?;+ let path = tmp_dir.path().join("data.parquet");+ let path = path.to_str().unwrap();+ ctx.sql(&format!(+ "COPY (SELECT value AS a, value % 1000 AS b, cast(value AS varchar) AS c \+ FROM generate_series(1, 1000000)) TO '{path}' STORED AS PARQUET"+ ))+ .await?+ .collect()+ .await?;+ ctx.register_parquet("t", path, ParquetReadOptions::default())+ .await?;++ let plan = ctx+ .sql("SELECT * FROM t")+ .await?+ .create_physical_plan()+ .await?;+ let start = Instant::now();+ collect(Arc::clone(&plan), ctx.task_ctx()).await?;+ let wall_time = start.elapsed();++ assert!(+ plan.is::<DataSourceExec>(),+ "expected a bare scan, got:\n{}",+ DisplayableExecutionPlan::new(plan.as_ref()).indent(true)+ );+ let elapsed_compute = Duration::from_nanos(+ plan.metrics()+ .unwrap()+ .aggregate_by_name()+ .elapsed_compute()+ .unwrap() as u64,+ );++ // Reading a local file is cheap next to decoding it, so compute should be+ // most of the wall time. A timer that misses decoding reports well under 1%.+ assert!(+ elapsed_compute * 4 > wall_time,+ "elapsed_compute {elapsed_compute:?} should be at least a quarter of the \+ scan's wall time {wall_time:?}"+ );+ Ok(())+}+// This test reproduces the behavior described in// https://github.com/apache/datafusion/issues/16684 where projection// pushdown with recursive CTEs could fail to remove unused columnsdiff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rsindex 77947bd8af..ae01d8862a 100644--- a/datafusion/datasource-parquet/src/push_decoder.rs+++ b/datafusion/datasource-parquet/src/push_decoder.rs@@ -430,16 +430,21 @@ impl PushDecoderStreamState {/// miri where `&mut self` creates a single opaque borrow that conflicts/// with `unfold`'s ownership across yield points.async fn transition(mut self) -> Option<(Result<RecordBatch>, Self)> {+ // Everything below is CPU work (decoding, row group pruning, building+ // readers, projection) except fetching byte ranges, so the timer runs+ // for the whole transition and is paused only across that await.+ // Cloning `Time` shares the underlying counter and keeps the guard+ // from borrowing `self`. The guard records on drop, which covers+ // every return.+ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let mut timer = elapsed_compute.timer();loop {// Step 1: drain a batch from the active reader if any.if let Some(reader) = self.active_reader.as_mut() {match reader.next() {Some(Ok(batch)) => {- let mut timer = self.baseline_metrics.elapsed_compute().timer();self.copy_arrow_reader_metrics();let result = self.project_batch(&batch);- timer.stop();- drop(timer);return Some((result, self));}Some(Err(e)) => {@@ -501,11 +506,14 @@ impl PushDecoderStreamState {let decoder = self.decoder.as_mut().expect("decoder present");match decoder.try_next_reader() {Ok(DecodeResult::NeedsData(ranges)) => {+ // I/O, not compute.+ timer.stop();let data = self.reader.get_byte_ranges(ranges.clone()).await.map_err(DataFusionError::from);+ timer.restart();match data {Ok(data) => {if let Err(e) = self