Skip to content

Apache DataFusion SQL Query Engine

fix: count Parquet decoding in the scan's elapsed_compute (#25486)

akashjainn0e3c398ee149
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs
index 2ca04ad6b3..67614cfbde 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -908,6 +908,64 @@ async fn parquet_explain_analyze() {
);
}
+/// The Parquet scan's `elapsed_compute` must cover decoding, which dominates a
+/// full scan, and not only the projection of batches that are already decoded.
+/// See <https://github.com/apache/datafusion/issues/18195>.
+#[tokio::test]
+async fn parquet_scan_elapsed_compute_includes_decoding() -> Result<()> {
+ use datafusion::datasource::source::DataSourceExec;
+ use datafusion_common::instant::Instant;
+ use std::time::Duration;
+
+ // A single partition keeps the scan on one thread, so its compute time
+ // cannot exceed the wall time of the query.
+ let ctx =
+ SessionContext::new_with_config(SessionConfig::new().with_target_partitions(1));
+ let tmp_dir = TempDir::new()?;
+ let path = tmp_dir.path().join("data.parquet");
+ let path = path.to_str().unwrap();
+ ctx.sql(&format!(
+ "COPY (SELECT value AS a, value % 1000 AS b, cast(value AS varchar) AS c \
+ FROM generate_series(1, 1000000)) TO '{path}' STORED AS PARQUET"
+ ))
+ .await?
+ .collect()
+ .await?;
+ ctx.register_parquet("t", path, ParquetReadOptions::default())
+ .await?;
+
+ let plan = ctx
+ .sql("SELECT * FROM t")
+ .await?
+ .create_physical_plan()
+ .await?;
+ let start = Instant::now();
+ collect(Arc::clone(&plan), ctx.task_ctx()).await?;
+ let wall_time = start.elapsed();
+
+ assert!(
+ plan.is::<DataSourceExec>(),
+ "expected a bare scan, got:\n{}",
+ DisplayableExecutionPlan::new(plan.as_ref()).indent(true)
+ );
+ let elapsed_compute = Duration::from_nanos(
+ plan.metrics()
+ .unwrap()
+ .aggregate_by_name()
+ .elapsed_compute()
+ .unwrap() as u64,
+ );
+
+ // Reading a local file is cheap next to decoding it, so compute should be
+ // most of the wall time. A timer that misses decoding reports well under 1%.
+ assert!(
+ elapsed_compute * 4 > wall_time,
+ "elapsed_compute {elapsed_compute:?} should be at least a quarter of the \
+ scan's wall time {wall_time:?}"
+ );
+ Ok(())
+}
+
// This test reproduces the behavior described in
// https://github.com/apache/datafusion/issues/16684 where projection
// pushdown with recursive CTEs could fail to remove unused columns
diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs
index 77947bd8af..ae01d8862a 100644
--- a/datafusion/datasource-parquet/src/push_decoder.rs
+++ b/datafusion/datasource-parquet/src/push_decoder.rs
@@ -430,16 +430,21 @@ impl PushDecoderStreamState {
/// miri where `&mut self` creates a single opaque borrow that conflicts
/// with `unfold`'s ownership across yield points.
async fn transition(mut self) -> Option<(Result<RecordBatch>, Self)> {
+ // Everything below is CPU work (decoding, row group pruning, building
+ // readers, projection) except fetching byte ranges, so the timer runs
+ // for the whole transition and is paused only across that await.
+ // Cloning `Time` shares the underlying counter and keeps the guard
+ // from borrowing `self`. The guard records on drop, which covers
+ // every return.
+ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();
+ let mut timer = elapsed_compute.timer();
loop {
// Step 1: drain a batch from the active reader if any.
if let Some(reader) = self.active_reader.as_mut() {
match reader.next() {
Some(Ok(batch)) => {
- let mut timer = self.baseline_metrics.elapsed_compute().timer();
self.copy_arrow_reader_metrics();
let result = self.project_batch(&batch);
- timer.stop();
- drop(timer);
return Some((result, self));
}
Some(Err(e)) => {
@@ -501,11 +506,14 @@ impl PushDecoderStreamState {
let decoder = self.decoder.as_mut().expect("decoder present");
match decoder.try_next_reader() {
Ok(DecodeResult::NeedsData(ranges)) => {
+ // I/O, not compute.
+ timer.stop();
let data = self
.reader
.get_byte_ranges(ranges.clone())
.await
.map_err(DataFusionError::from);
+ timer.restart();
match data {
Ok(data) => {
if let Err(e) = self