Apache DataFusion SQL Query Engine
fix: fix elapsed_compute metric in ParquetSink to report encoding time only (#21825)
Fred Thomas3aefba75be1e
diff --git a/Cargo.lock b/Cargo.lockindex 24245e4eb4..9177551851 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2383,6 +2383,7 @@ dependencies = ["indexmap 2.14.0","itertools 0.14.0","parking_lot",+ "pin-project","rand 0.9.4",]diff --git a/Cargo.toml b/Cargo.tomlindex 0c768199c9..3773421126 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -183,6 +183,7 @@ parquet = { version = "58.1.0", default-features = false, features = [] }pbjson = { version = "0.9.0" }pbjson-types = "0.9"+pin-project = "1"# Should match arrow-flight's version of prost.prost = "0.14.1"rand = "0.9"diff --git a/datafusion/core/src/dataframe/parquet.rs b/datafusion/core/src/dataframe/parquet.rsindex ccd5766f0a..36a60989b2 100644--- a/datafusion/core/src/dataframe/parquet.rs+++ b/datafusion/core/src/dataframe/parquet.rs@@ -558,6 +558,66 @@ mod tests {assert_eq!(metric_usize(&aggregated, "rows_written"), 50);assert!(metric_usize(&aggregated, "bytes_written") > 0);+ assert!(+ metric_usize(&aggregated, "elapsed_compute") > 0,+ "expected elapsed_compute > 0 on parallel path"+ );++ Ok(())+ }++ /// Test that ParquetSink reports a non-zero elapsed_compute on the sequential+ /// write path (allow_single_file_parallelism = false), where elapsed_compute+ /// is computed as total_write_time - io_time via TimingWriter.+ #[tokio::test]+ async fn test_parquet_sink_metrics_sequential() -> Result<()> {+ use arrow::array::Int32Array;+ use arrow::datatypes::{DataType, Field, Schema};+ use arrow::record_batch::RecordBatch;+ use datafusion_execution::TaskContext;++ use futures::TryStreamExt;++ let ctx = SessionContext::new();+ ctx.sql("SET datafusion.execution.parquet.allow_single_file_parallelism = false")+ .await?+ .collect()+ .await?;++ let tmp_dir = TempDir::new()?;+ let output_path = tmp_dir.path().join("metrics_sequential.parquet");+ let output_path_str = output_path.to_str().unwrap();++ let schema =+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));+ let ids: Vec<i32> = (0..50).collect();+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(ids))],+ )?;+ ctx.register_batch("source_seq", batch)?;++ let df = ctx+ .sql(&format!(+ "COPY source_seq TO '{output_path_str}' STORED AS PARQUET"+ ))+ .await?;+ let plan = df.create_physical_plan().await?;+ let task_ctx = Arc::new(TaskContext::from(&ctx.state()));+ let stream = plan.execute(0, task_ctx)?;+ let _batches: Vec<_> = stream.try_collect().await?;++ let metrics = plan+ .metrics()+ .expect("DataSinkExec should return metrics from ParquetSink");+ let aggregated = metrics.aggregate_by_name();++ assert_eq!(metric_usize(&aggregated, "rows_written"), 50);+ assert!(metric_usize(&aggregated, "bytes_written") > 0);+ assert!(+ metric_usize(&aggregated, "elapsed_compute") > 0,+ "expected elapsed_compute > 0 on sequential path"+ );Ok(())}diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex 7dda7b1b12..d97077d609 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -55,7 +55,8 @@ use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::dml::InsertOp;use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};use datafusion_physical_plan::metrics::{- ExecutionPlanMetricsSet, MetricBuilder, MetricCategory, MetricsSet,+ ElapsedComputeFutureExt, ExecutionPlanMetricsSet, MetricBuilder, MetricCategory,+ MetricsSet, Time,};use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};use datafusion_session::Session;@@ -1340,8 +1341,6 @@ impl FileSink for ParquetSink {.global_counter("bytes_written");let elapsed_compute = MetricBuilder::new(&self.metrics).elapsed_compute(0);- let write_start = datafusion_common::instant::Instant::now();-let parquet_opts = &self.parquet_options;let mut file_write_tasks: JoinSet<@@ -1375,17 +1374,20 @@ impl FileSink for ParquetSink {.await?;let reservation = MemoryConsumer::new(format!("ParquetSink[{path}]")).register(context.memory_pool());- file_write_tasks.spawn(async move {- while let Some(batch) = rx.recv().await {- writer.write(&batch).await?;- reservation.try_resize(writer.memory_size())?;+ file_write_tasks.spawn(+ async move {+ while let Some(batch) = rx.recv().await {+ writer.write(&batch).await?;+ reservation.try_resize(writer.memory_size())?;+ }+ let parquet_meta_data = writer+ .close()+ .await+ .map_err(|e| DataFusionError::ParquetError(Box::new(e)))?;+ Ok((path, parquet_meta_data))}- let parquet_meta_data = writer- .close()- .await- .map_err(|e| DataFusionError::ParquetError(Box::new(e)))?;- Ok((path, parquet_meta_data))- });+ .with_elapsed_compute(elapsed_compute.clone()),+ );} else {let writer = ObjectWriterBuilder::new(// Parquet files as a whole are never compressed, since they@@ -1402,20 +1404,20 @@ impl FileSink for ParquetSink {.objectstore_writer_buffer_size,)).build()?;- let schema = get_writer_schema(&self.config);- let props = parquet_props.clone();- let skip_arrow_metadata = self.parquet_options.global.skip_arrow_metadata;- let parallel_options_clone = parallel_options.clone();- let pool = Arc::clone(context.memory_pool());+ let ctx = ParquetFileWriteContext {+ schema: get_writer_schema(&self.config),+ props: Arc::new(parquet_props),+ skip_arrow_metadata: self.parquet_options.global.skip_arrow_metadata,+ parallel_options: Arc::new(parallel_options.clone()),+ pool: Arc::clone(context.memory_pool()),+ };+ let encoding_time = elapsed_compute.clone();file_write_tasks.spawn(async move {let parquet_meta_data = output_single_parquet_file_parallelized(writer,rx,- schema,- &props,- skip_arrow_metadata,- parallel_options_clone,- pool,+ ctx,+ encoding_time,).await?;Ok((path, parquet_meta_data))@@ -1456,8 +1458,6 @@ impl FileSink for ParquetSink {.await.map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;- elapsed_compute.add_elapsed(write_start);-Ok(rows_written_counter.value() as u64)}}@@ -1487,8 +1487,10 @@ async fn column_serializer_task(mut rx: Receiver<ArrowLeafColumn>,mut writer: ArrowColumnWriter,reservation: MemoryReservation,+ encoding_time: Time,) -> Result<(ArrowColumnWriter, MemoryReservation)> {while let Some(col) = rx.recv().await {+ let _timer = encoding_time.timer();writer.write(&col)?;reservation.try_resize(writer.memory_size())?;}@@ -1505,6 +1507,7 @@ fn spawn_column_parallel_row_group_writer(col_writers: Vec<ArrowColumnWriter>,max_buffer_size: usize,pool: &Arc<dyn MemoryPool>,+ encoding_time: &Time,) -> Result<(Vec<ColumnWriterTask>, Vec<ColSender>)> {let num_columns = col_writers.len();@@ -1522,6 +1525,7 @@ fn spawn_column_parallel_row_group_writer(receive_array,writer,reservation,+ encoding_time.clone(),));col_writer_tasks.push(task);}@@ -1536,6 +1540,22 @@ struct ParallelParquetWriterOptions {max_buffered_record_batches_per_stream: usize,}+/// Write configuration inputs shared across all parallel tasks that encode a+/// single Parquet file. These values are invariant for the duration of one file+/// write and do not change per row-group or per column.+///+/// Separating these from per-call parameters (`object_store_writer`, `data`,+/// `encoding_time`) keeps the deep parallel call chain below the argument-count+/// limit without mixing configuration with runtime state.+#[derive(Clone)]+struct ParquetFileWriteContext {+ schema: Arc<Schema>,+ props: Arc<WriterProperties>,+ skip_arrow_metadata: bool,+ parallel_options: Arc<ParallelParquetWriterOptions>,+ pool: Arc<dyn MemoryPool>,+}+/// This is the return type of calling [ArrowColumnWriter].close() on each column/// i.e. the Vec of encoded columns which can be appended to a row grouptype RBStreamSerializeResult = Result<(Vec<ArrowColumnChunk>, MemoryReservation, usize)>;@@ -1570,6 +1590,7 @@ fn spawn_rg_join_and_finalize_task(column_writer_tasks: Vec<ColumnWriterTask>,rg_rows: usize,pool: &Arc<dyn MemoryPool>,+ encoding_time: Time,) -> SpawnedTask<RBStreamSerializeResult> {let rg_reservation =MemoryConsumer::new("ParquetSink(SerializedRowGroupWriter)").register(pool);@@ -1584,6 +1605,7 @@ fn spawn_rg_join_and_finalize_task(.map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;let encoded_size = writer.get_estimated_total_bytes();rg_reservation.grow(encoded_size);+ let _timer = encoding_time.timer();finalized_rg.push(writer.close()?);}@@ -1603,21 +1625,25 @@ fn spawn_parquet_parallel_serialization_task(row_group_writer_factory: ArrowRowGroupWriterFactory,mut data: Receiver<RecordBatch>,serialize_tx: Sender<SpawnedTask<RBStreamSerializeResult>>,- schema: Arc<Schema>,- writer_props: Arc<WriterProperties>,- parallel_options: Arc<ParallelParquetWriterOptions>,- pool: Arc<dyn MemoryPool>,+ ctx: ParquetFileWriteContext,+ encoding_time: Time,) -> SpawnedTask<Result<(), DataFusionError>> {SpawnedTask::spawn(async move {- let max_buffer_rb = parallel_options.max_buffered_record_batches_per_stream;- let max_row_group_rows = writer_props+ let max_buffer_rb = ctx.parallel_options.max_buffered_record_batches_per_stream;+ let max_row_group_rows = ctx+ .props.max_row_group_row_count().unwrap_or(DEFAULT_MAX_ROW_GROUP_ROW_COUNT);let mut row_group_index = 0;let col_writers =row_group_writer_factory.create_column_writers(row_group_index)?;let (mut column_writer_handles, mut col_array_channels) =- spawn_column_parallel_row_group_writer(col_writers, max_buffer_rb, &pool)?;+ spawn_column_parallel_row_group_writer(+ col_writers,+ max_buffer_rb,+ &ctx.pool,+ &encoding_time,+ )?;let mut current_rg_rows = 0;while let Some(mut rb) = data.recv().await {@@ -1629,7 +1655,7 @@ fn spawn_parquet_parallel_serialization_task(send_arrays_to_col_writers(&col_array_channels,&rb,- Arc::clone(&schema),+ Arc::clone(&ctx.schema),).await?;current_rg_rows += rb.num_rows();@@ -1640,7 +1666,7 @@ fn spawn_parquet_parallel_serialization_task(send_arrays_to_col_writers(&col_array_channels,&a,- Arc::clone(&schema),+ Arc::clone(&ctx.schema),).await?;@@ -1651,7 +1677,8 @@ fn spawn_parquet_parallel_serialization_task(let finalize_rg_task = spawn_rg_join_and_finalize_task(column_writer_handles,max_row_group_rows,- &pool,+ &ctx.pool,+ encoding_time.clone(),);// Do not surface error from closed channel (means something@@ -1670,7 +1697,8 @@ fn spawn_parquet_parallel_serialization_task(spawn_column_parallel_row_group_writer(col_writers,max_buffer_rb,- &pool,+ &ctx.pool,+ &encoding_time,)?;}}@@ -1682,7 +1710,8 @@ fn spawn_parquet_parallel_serialization_task(let finalize_rg_task = spawn_rg_join_and_finalize_task(column_writer_handles,current_rg_rows,- &pool,+ &ctx.pool,+ encoding_time.clone(),);// Do not surface error from closed channel (means something@@ -1749,37 +1778,32 @@ async fn concatenate_parallel_row_groups(async fn output_single_parquet_file_parallelized(object_store_writer: Box<dyn AsyncWrite + Send + Unpin>,data: Receiver<RecordBatch>,- output_schema: Arc<Schema>,- parquet_props: &WriterProperties,- skip_arrow_metadata: bool,- parallel_options: ParallelParquetWriterOptions,- pool: Arc<dyn MemoryPool>,+ ctx: ParquetFileWriteContext,+ encoding_time: Time,) -> Result<ParquetMetaData> {- let max_rowgroups = parallel_options.max_parallel_row_groups;+ let max_rowgroups = ctx.parallel_options.max_parallel_row_groups;// Buffer size of this channel limits maximum number of RowGroups being worked on in parallellet (serialize_tx, serialize_rx) =mpsc::channel::<SpawnedTask<RBStreamSerializeResult>>(max_rowgroups);- let arc_props = Arc::new(parquet_props.clone());let merged_buff = SharedBuffer::new(INITIAL_BUFFER_BYTES);let options = ArrowWriterOptions::new()- .with_properties(parquet_props.clone())- .with_skip_arrow_metadata(skip_arrow_metadata);+ .with_properties((*ctx.props).clone())+ .with_skip_arrow_metadata(ctx.skip_arrow_metadata);let writer = ArrowWriter::try_new_with_options(merged_buff.clone(),- Arc::clone(&output_schema),+ Arc::clone(&ctx.schema),options,)?;let (writer, row_group_writer_factory) = writer.into_serialized_writer()?;+ let pool = Arc::clone(&ctx.pool);let launch_serialization_task = spawn_parquet_parallel_serialization_task(row_group_writer_factory,data,serialize_tx,- Arc::clone(&output_schema),- Arc::clone(&arc_props),- parallel_options.into(),- Arc::clone(&pool),+ ctx,+ encoding_time,);let parquet_meta_data = concatenate_parallel_row_groups(writer,diff --git a/datafusion/physical-expr-common/Cargo.toml b/datafusion/physical-expr-common/Cargo.tomlindex 119be36234..0e4748b81d 100644--- a/datafusion/physical-expr-common/Cargo.toml+++ b/datafusion/physical-expr-common/Cargo.toml@@ -49,6 +49,7 @@ hashbrown = { workspace = true }indexmap = { workspace = true }itertools = { workspace = true }parking_lot = { workspace = true }+pin-project = { workspace = true }[dev-dependencies]criterion = { workspace = true }diff --git a/datafusion/physical-expr-common/src/metrics/elapsed_compute.rs b/datafusion/physical-expr-common/src/metrics/elapsed_compute.rsnew file mode 100644index 0000000000..f792f9f46b--- /dev/null+++ b/datafusion/physical-expr-common/src/metrics/elapsed_compute.rs@@ -0,0 +1,101 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::future::Future;+use std::pin::Pin;+use std::task::{Context, Poll};+use std::time::Duration;++use datafusion_common::instant::Instant;+use pin_project::{pin_project, pinned_drop};++use super::Time;++/// Wraps any [`Future`] and accumulates the wall-clock time spent inside+/// each [`Future::poll`] call into `elapsed_compute`. Everything that+/// executes synchronously within a `poll()` scope is measured — including+/// CPU-bound work, memory copies, and any blocking the future performs+/// before returning. Time between polls (when the runtime has suspended the+/// future waiting for I/O, a channel, or a waker) is not measured.+///+/// For futures that mix synchronous CPU work with async I/O this gives a+/// good approximation of CPU time: async I/O causes the future to yield+/// (`Poll::Pending`), so the I/O latency is excluded automatically.+///+/// Note: uses `pin-project` rather than `pin-project-lite` in order to+/// support `PinnedDrop`, which ensures accumulated time is flushed even+/// if the future is cancelled (dropped before completion).+#[pin_project(PinnedDrop)]+pub struct ElapsedComputeFuture<T> {+ #[pin]+ inner: T,+ /// Local accumulator: elapsed time is collected here during each `poll()`+ /// and only flushed to `elapsed_compute` on completion (`Poll::Ready`) or+ /// on drop (`PinnedDrop`). Keeping a separate local `Duration` avoids+ /// performing an atomic operation on every `poll()` call, at the cost of+ /// the reported metric value being unavailable until the future finishes.+ curr: Duration,+ elapsed_compute: Time,+}++#[pinned_drop]+impl<T> PinnedDrop for ElapsedComputeFuture<T> {+ fn drop(self: Pin<&mut Self>) {+ if self.curr > Duration::default() {+ let self_projected = self.project();+ self_projected+ .elapsed_compute+ .add_duration(*self_projected.curr);+ }+ }+}++impl<O, F: Future<Output = O>> Future for ElapsedComputeFuture<F> {+ type Output = O;++ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {+ let self_projected = self.project();+ let start = Instant::now();+ let result = self_projected.inner.poll(cx);+ *self_projected.curr += start.elapsed();+ if result.is_ready() {+ self_projected+ .elapsed_compute+ .add_duration(*self_projected.curr);+ *self_projected.curr = Duration::default();+ }+ result+ }+}++/// Extension trait that wraps any [`Future`] with [`ElapsedComputeFuture`].+pub trait ElapsedComputeFutureExt: Future + Sized {+ /// Wraps this future so that the time spent inside each [`Future::poll`]+ /// call is accumulated into `elapsed_compute`. See [`ElapsedComputeFuture`]+ /// for a full description of what is and is not measured.+ fn with_elapsed_compute(self, elapsed_compute: Time) -> ElapsedComputeFuture<Self>;+}++impl<O, F: Future<Output = O>> ElapsedComputeFutureExt for F {+ fn with_elapsed_compute(self, elapsed_compute: Time) -> ElapsedComputeFuture<Self> {+ ElapsedComputeFuture {+ inner: self,+ curr: Duration::default(),+ elapsed_compute,+ }+ }+}diff --git a/datafusion/physical-expr-common/src/metrics/mod.rs b/datafusion/physical-expr-common/src/metrics/mod.rsindex 285e856635..eecd8cfabd 100644--- a/datafusion/physical-expr-common/src/metrics/mod.rs+++ b/datafusion/physical-expr-common/src/metrics/mod.rs@@ -20,6 +20,7 @@mod baseline;mod builder;mod custom;+mod elapsed_compute;mod expression;mod value;@@ -38,6 +39,7 @@ use std::{pub use baseline::{BaselineMetrics, RecordOutput, SpillMetrics, SplitMetrics};pub use builder::MetricBuilder;pub use custom::CustomMetricValue;+pub use elapsed_compute::{ElapsedComputeFuture, ElapsedComputeFutureExt};pub use expression::ExpressionEvaluatorMetrics;pub use value::{Count, Gauge, MetricValue, PruningMetrics, RatioMergeStrategy, RatioMetrics,