Apache DataFusion SQL Query Engine
Report peak MemoryPool reservation per query in benchmarks (#23985)
Adrian Garcia Badaraccof398301748b4
diff --git a/Cargo.lock b/Cargo.lockindex a41734e064..451ff70b1c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1780,6 +1780,7 @@ name = "datafusion-benchmarks"version = "54.1.0"dependencies = ["arrow",+ "arrow-buffer","async-trait","bytes","clap",@@ -1787,6 +1788,7 @@ dependencies = ["datafusion","datafusion-common","datafusion-common-runtime",+ "datafusion-execution","datafusion-proto","env_logger","futures",diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.tomlindex 5dae70761f..282b27e481 100644--- a/benchmarks/Cargo.toml+++ b/benchmarks/Cargo.toml@@ -64,6 +64,11 @@ tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }tokio-util = { version = "0.7.17" }[dev-dependencies]+# `pool`/`arrow_buffer_pool` are enabled only for tests, so the benchmark+# binaries are built exactly as before. They let `memory_pool`'s tests cover+# Arrow-side reservations reaching the pool via `ArrowMemoryPool`.+arrow-buffer = { workspace = true, features = ["pool"] }+datafusion-execution = { workspace = true, features = ["arrow_buffer_pool"] }datafusion-proto = { workspace = true, features = ["parquet"] }tempfile = { workspace = true }diff --git a/benchmarks/README.md b/benchmarks/README.mdindex 34c67e5151..b6a7705cf9 100644--- a/benchmarks/README.md+++ b/benchmarks/README.md@@ -483,6 +483,14 @@ Your benchmark should create and use an instance of `BenchmarkRun` defined in `b- Call its `start_new_case` method with a string that will appear in the "Query" column of thecompare output.- Use `write_iter` to record elapsed times for the behavior you're benchmarking.+- Call `set_memory_pool` with the `RuntimeEnv`'s memory pool (`ctx.runtime_env().memory_pool`),+ and again for each new runtime if your benchmark builds one per query. Each case then reports a+ `pool_peak_bytes` field: the peak `MemoryPool` reservation reached while running it, which is the+ largest value across that case's iterations. The field is omitted when the benchmark runs without+ `--memory-limit`, since no pool is installed to record. Comparing it against the peak RSS printed+ by `print_memory_stats` shows how much of the run's memory the pool actually accounted for; the+ pool only tracks the "large" allocations that scale with input size, so the two are expected to+ differ.- When all cases are done, call the `BenchmarkRun`'s `maybe_write_json` method, giving it the valueof the `--output` structopt field on `RunOpt`.diff --git a/benchmarks/src/bin/benchmark_runner.rs b/benchmarks/src/bin/benchmark_runner.rsindex 71b3b1b9e0..c7a16086c9 100644--- a/benchmarks/src/bin/benchmark_runner.rs+++ b/benchmarks/src/bin/benchmark_runner.rs@@ -299,6 +299,9 @@ async fn run_simple_benchmark(let case_name = benchmark_case_name(benchmark);+ // Each case gets its own `SessionContext`, so hand over its pool before the+ // case starts.+ run.set_memory_pool(&ctx.runtime_env().memory_pool);run.start_new_case(&case_name);for iteration in 0..config.common.iterations {@@ -312,7 +315,7 @@ async fn run_simple_benchmark(run.write_iter(elapsed, row_count);}- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(())}diff --git a/benchmarks/src/bin/external_aggr.rs b/benchmarks/src/bin/external_aggr.rsindex 42f25c2cb0..226a619192 100644--- a/benchmarks/src/bin/external_aggr.rs+++ b/benchmarks/src/bin/external_aggr.rs@@ -39,7 +39,9 @@ use datafusion::execution::runtime_env::RuntimeEnvBuilder;use datafusion::physical_plan::display::DisplayableExecutionPlan;use datafusion::physical_plan::{collect, displayable};use datafusion::prelude::*;-use datafusion_benchmarks::util::{BenchmarkRun, CommonOpt, QueryResult};+use datafusion_benchmarks::util::{+ BenchmarkRun, CommonOpt, PeakRecordingPool, QueryResult,+};use datafusion_common::instant::Instant;use datafusion_common::utils::get_available_parallelism;use datafusion_common::{DEFAULT_PARQUET_EXTENSION, exec_err};@@ -169,7 +171,7 @@ impl ExternalAggrConfig {));let query_results = self- .benchmark_query(query_id, mem_limit, mem_pool_type)+ .benchmark_query(query_id, mem_limit, mem_pool_type, &mut benchmark_run).await?;for iter in query_results {benchmark_run.write_iter(iter.elapsed, iter.row_count);@@ -182,11 +184,15 @@ impl ExternalAggrConfig {}/// Benchmark query `query_id` in `AGGR_QUERIES`+ ///+ /// `benchmark_run` is handed this query's runtime, which is built here+ /// because each query runs under its own memory limit.async fn benchmark_query(&self,query_id: usize,mem_limit: u64,mem_pool_type: &str,+ benchmark_run: &mut BenchmarkRun,) -> Result<Vec<QueryResult>> {let query_name =format!("Q{query_id}({})", human_readable_size(mem_limit as usize));@@ -198,6 +204,12 @@ impl ExternalAggrConfig {return exec_err!("Invalid memory pool type: {}", mem_pool_type);}};+ // This benchmark builds its pool directly rather than going through+ // `CommonOpt::runtime_env_builder`, so it has to install the recorder+ // itself to report a peak.+ let memory_pool: Arc<dyn MemoryPool> =+ Arc::new(PeakRecordingPool::new(memory_pool));+ benchmark_run.set_memory_pool(&memory_pool);let runtime_env = RuntimeEnvBuilder::new().with_memory_pool(memory_pool).build_arc()?;diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rsindex 70aaeb7d2d..a2e65aa561 100644--- a/benchmarks/src/clickbench.rs+++ b/benchmarks/src/clickbench.rs@@ -213,6 +213,7 @@ impl RunOpt {self.register_hits(&ctx).await?;let mut benchmark_run = BenchmarkRun::new();+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);for query_id in query_range {let query_path = get_query_path(&self.queries_path, query_id);let Some(sql) = get_query_sql(&query_path)? else {@@ -278,7 +279,7 @@ impl RunOpt {println!("Query {query_id} avg time: {avg:.2} ms");// Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended)- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(query_results)}diff --git a/benchmarks/src/dict.rs b/benchmarks/src/dict.rsindex f8451715ea..e04b5f816a 100644--- a/benchmarks/src/dict.rs+++ b/benchmarks/src/dict.rs@@ -333,6 +333,7 @@ impl RunOpt {let rt = self.common.build_runtime()?;let ctx = SessionContext::new_with_config_rt(config, rt);let mut benchmark_run = BenchmarkRun::new();+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);for query_id in query_range {let query = &DICTIONARY_QUERIES[query_id - 1];diff --git a/benchmarks/src/h2o.rs b/benchmarks/src/h2o.rsindex 8b6e04932c..feb4bf2fa1 100644--- a/benchmarks/src/h2o.rs+++ b/benchmarks/src/h2o.rs@@ -109,6 +109,7 @@ impl RunOpt {let iterations = self.common.iterations;let mut benchmark_run = BenchmarkRun::new();+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);for query_id in query_range {benchmark_run.start_new_case(&format!("Query {query_id}"));let sql = queries.get_query(query_id)?;@@ -131,7 +132,7 @@ impl RunOpt {println!("Query {query_id} avg time: {avg:.2} ms");// Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended)- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);if self.common.debug {ctx.sql(sql)diff --git a/benchmarks/src/hj.rs b/benchmarks/src/hj.rsindex 9a91e2714a..4f97b24d0f 100644--- a/benchmarks/src/hj.rs+++ b/benchmarks/src/hj.rs@@ -564,6 +564,7 @@ impl RunOpt {}let mut benchmark_run = BenchmarkRun::new();+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);for query_id in query_range {let query_index = query_id - 1;diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rsindex a8e2028887..5822bbcb0d 100644--- a/benchmarks/src/imdb/run.rs+++ b/benchmarks/src/imdb/run.rs@@ -295,7 +295,7 @@ impl RunOpt {let mut benchmark_run = BenchmarkRun::new();for query_id in query_range {benchmark_run.start_new_case(&format!("Query {query_id}"));- let query_run = self.benchmark_query(query_id).await?;+ let query_run = self.benchmark_query(query_id, &mut benchmark_run).await?;for iter in query_run {benchmark_run.write_iter(iter.elapsed, iter.row_count);}@@ -304,7 +304,13 @@ impl RunOpt {Ok(())}- async fn benchmark_query(&self, query_id: usize) -> Result<Vec<QueryResult>> {+ /// `benchmark_run` is handed this query's runtime, which is built here so+ /// each query gets a pool of its own.+ async fn benchmark_query(+ &self,+ query_id: usize,+ benchmark_run: &mut BenchmarkRun,+ ) -> Result<Vec<QueryResult>> {let mut config = self.common.config()?@@ -314,6 +320,7 @@ impl RunOpt {self.hash_join_buffering_capacity;let rt = self.common.build_runtime()?;let ctx = SessionContext::new_with_config_rt(config, rt);+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);// register tablesself.register_tables(&ctx).await?;@@ -348,7 +355,7 @@ impl RunOpt {println!("Query {query_id} avg time: {avg:.2} ms");// Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended)- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(query_results)}diff --git a/benchmarks/src/nlj.rs b/benchmarks/src/nlj.rsindex 361cc35ec2..485ee069d1 100644--- a/benchmarks/src/nlj.rs+++ b/benchmarks/src/nlj.rs@@ -211,6 +211,7 @@ impl RunOpt {let ctx = SessionContext::new_with_config_rt(config, rt);let mut benchmark_run = BenchmarkRun::new();+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);for query_id in query_range {let query_index = query_id - 1; // Convert 1-based to 0-based indexdiff --git a/benchmarks/src/smj.rs b/benchmarks/src/smj.rsindex 3d173b7116..9282f72c2f 100644--- a/benchmarks/src/smj.rs+++ b/benchmarks/src/smj.rs@@ -550,6 +550,7 @@ impl RunOpt {let ctx = SessionContext::new_with_config_rt(config, rt);let mut benchmark_run = BenchmarkRun::new();+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);for query_id in query_range {let query_index = query_id - 1; // Convert 1-based to 0-based indexdiff --git a/benchmarks/src/sort_pushdown.rs b/benchmarks/src/sort_pushdown.rsindex 86f1c0f5c1..77f889e702 100644--- a/benchmarks/src/sort_pushdown.rs+++ b/benchmarks/src/sort_pushdown.rs@@ -137,7 +137,7 @@ impl RunOpt {for query_id in query_ids {benchmark_run.start_new_case(&format!("{query_id}"));- let query_results = self.benchmark_query(query_id).await;+ let query_results = self.benchmark_query(query_id, &mut benchmark_run).await;match query_results {Ok(query_results) => {for iter in query_results {@@ -156,7 +156,13 @@ impl RunOpt {Ok(())}- async fn benchmark_query(&self, query_id: usize) -> Result<Vec<QueryResult>> {+ /// `benchmark_run` is handed this query's runtime, which is built here so+ /// each query gets a pool of its own.+ async fn benchmark_query(+ &self,+ query_id: usize,+ benchmark_run: &mut BenchmarkRun,+ ) -> Result<Vec<QueryResult>> {let sql = self.load_query(query_id)?;let config = self.common.config()?;@@ -168,6 +174,7 @@ impl RunOpt {.with_default_features().build();let ctx = SessionContext::from(state);+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);self.register_tables(&ctx).await?;@@ -191,7 +198,7 @@ impl RunOpt {let avg = millis.iter().sum::<f64>() / millis.len() as f64;println!("Query {query_id} avg time: {avg:.2} ms");- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(query_results)}diff --git a/benchmarks/src/sort_tpch.rs b/benchmarks/src/sort_tpch.rsindex 2182d1a383..d5f81c04a3 100644--- a/benchmarks/src/sort_tpch.rs+++ b/benchmarks/src/sort_tpch.rs@@ -187,7 +187,7 @@ impl RunOpt {for query_id in query_range {benchmark_run.start_new_case(&format!("{query_id}"));- let query_results = self.benchmark_query(query_id).await;+ let query_results = self.benchmark_query(query_id, &mut benchmark_run).await;match query_results {Ok(query_results) => {for iter in query_results {@@ -207,7 +207,14 @@ impl RunOpt {}/// Benchmark query `query_id` in `SORT_QUERIES`- async fn benchmark_query(&self, query_id: usize) -> Result<Vec<QueryResult>> {+ ///+ /// `benchmark_run` is handed this query's runtime, which is built here so+ /// each query gets a pool of its own.+ async fn benchmark_query(+ &self,+ query_id: usize,+ benchmark_run: &mut BenchmarkRun,+ ) -> Result<Vec<QueryResult>> {let config = self.common.config()?;let rt = self.common.build_runtime()?;let state = SessionStateBuilder::new()@@ -216,6 +223,7 @@ impl RunOpt {.with_default_features().build();let ctx = SessionContext::from(state);+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);// register tablesself.register_tables(&ctx).await?;@@ -250,7 +258,7 @@ impl RunOpt {println!("Query {query_id} avg time: {avg:.2} ms");// Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended)- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(query_results)}diff --git a/benchmarks/src/sql_benchmark_runner.rs b/benchmarks/src/sql_benchmark_runner.rsindex f1cb3ad2f7..420e780c64 100644--- a/benchmarks/src/sql_benchmark_runner.rs+++ b/benchmarks/src/sql_benchmark_runner.rs@@ -110,7 +110,7 @@ fn run_criterion_benchmark(match result {Ok(()) => {- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(())}Err(payload) => Err(panic_payload_to_error(payload.as_ref())),diff --git a/benchmarks/src/tpcds/run.rs b/benchmarks/src/tpcds/run.rsindex 2e0274c935..3eaaf172c0 100644--- a/benchmarks/src/tpcds/run.rs+++ b/benchmarks/src/tpcds/run.rs@@ -226,6 +226,7 @@ impl RunOpt {self.hash_join_buffering_capacity;let rt = self.common.build_runtime()?;let ctx = SessionContext::new_with_config_rt(config, rt);+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);// register tablesself.register_tables(&ctx).await?;@@ -290,7 +291,7 @@ impl RunOpt {println!("Query {query_id} avg time: {avg:.2} ms");// Print memory stats using mimalloc (only when compiled with --features mimalloc_extended)- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(query_results)}diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rsindex 422bcec9ea..47edfbac4b 100644--- a/benchmarks/src/tpch/run.rs+++ b/benchmarks/src/tpch/run.rs@@ -137,6 +137,7 @@ impl RunOpt {self.hash_join_buffering_capacity;let rt = self.common.build_runtime()?;let ctx = SessionContext::new_with_config_rt(config, rt);+ benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool);// register tablesself.register_tables(&ctx).await?;let scale_factor = self.scale_factor()?;@@ -208,7 +209,7 @@ impl RunOpt {println!("Query {query_id} avg time: {avg:.2} ms");// Print memory stats using mimalloc (only when compiled with --features mimalloc_extended)- print_memory_stats();+ print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(query_results)}diff --git a/benchmarks/src/util/memory.rs b/benchmarks/src/util/memory.rsindex 11b96ef227..2b186c79c3 100644--- a/benchmarks/src/util/memory.rs+++ b/benchmarks/src/util/memory.rs@@ -15,8 +15,34 @@// specific language governing permissions and limitations// under the License.-/// Print Peak RSS, Peak Commit, Page Faults based on mimalloc api-pub fn print_memory_stats() {+use datafusion::execution::memory_pool::MemoryPool;++use super::PeakRecordingPool;++/// Print Peak RSS, Peak Commit, Page Faults based on mimalloc api, followed by+/// the peak reservation of `memory_pool` when a memory limit was configured.+pub fn print_memory_stats(memory_pool: &dyn MemoryPool) {+ print_allocator_stats();+ print_pool_stats(memory_pool);+}++/// Print the peak reservation `memory_pool` has seen.+///+/// Prints nothing when the benchmark ran without a memory limit, since no+/// [`PeakRecordingPool`] was installed to record. Comparing this against the+/// peak RSS above shows how much of a run's memory the pool actually accounted+/// for — DataFusion only tracks the "large" allocations that scale with input+/// size, so the two are expected to differ.+fn print_pool_stats(memory_pool: &dyn MemoryPool) {+ if let Some(recorder) = PeakRecordingPool::from_pool(memory_pool) {+ println!(+ "Peak pool reserved: {}",+ datafusion_common::human_readable_size(recorder.max_reserved())+ );+ }+}++fn print_allocator_stats() {#[cfg(all(feature = "mimalloc", feature = "mimalloc_extended"))]{use datafusion_common::human_readable_size;diff --git a/benchmarks/src/util/memory_pool.rs b/benchmarks/src/util/memory_pool.rsnew file mode 100644index 0000000000..a3606ca0a7--- /dev/null+++ b/benchmarks/src/util/memory_pool.rs@@ -0,0 +1,381 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Records the peak [`MemoryPool`] reservation reached during a benchmark.+//!+//! DataFusion's [`MemoryPool`] deliberately accounts for only the "large"+//! allocations that scale with input size; intermediate batches flowing between+//! operators are assumed to be small and are left untracked. The [`MemoryPool`]+//! documentation therefore advises reserving "some overhead (e.g. 10%)" on top+//! of the configured limit.+//!+//! Nothing reports what that overhead actually is, because the peak reservation+//! itself is never recorded — [`MemoryPool::reserved`] is a live value that has+//! usually fallen back to zero by the time a query finishes. This module records+//! the high-water mark so benchmarks can emit it alongside the peak RSS that+//! [`print_memory_stats`] already prints, making the gap between the two+//! measurable.+//!+//! This is measurement only: nothing here enforces a relationship between the+//! two numbers.+//!+//! What lands in the peak is whatever the pool accounts for, so this follows+//! the accounting rather than fixing it in place. Arrow-side reservations made+//! through `ArrowMemoryPool` are included, because that adapter grows a+//! DataFusion reservation against the pool it wraps; nothing claims buffers+//! today, but the peak picks it up when something does.+//!+//! [`print_memory_stats`]: super::print_memory_stats++use std::{+ fmt::{Debug, Display, Formatter},+ sync::{+ Arc,+ atomic::{AtomicUsize, Ordering},+ },+};++use datafusion::execution::memory_pool::{+ MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation,+};+use datafusion_common::Result;++/// Wraps a [`MemoryPool`], recording the high-water mark of+/// [`MemoryPool::reserved`] as reservations come and go.+///+/// Every method delegates to the wrapped pool, so wrapping does not change how+/// memory is granted, limited, or reported. The one thing it does change is+/// downcasting: `rt.memory_pool.downcast_ref::<FairSpillPool>()` now finds this+/// wrapper instead of the pool it wraps. Nothing in the benchmarks relies on+/// that, and [`Self::from_pool`] uses the same mechanism to find the recorder.+///+/// Both high-water marks are held per instance, so a benchmark that builds a+/// fresh runtime per query gets a reading scoped to that query without any+/// coordination.+///+/// # Example+///+/// ```+/// # use std::sync::Arc;+/// # use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool};+/// # use datafusion_benchmarks::util::PeakRecordingPool;+/// let recording = Arc::new(PeakRecordingPool::new(Arc::new(GreedyMemoryPool::new(1024))));+/// let pool: Arc<dyn MemoryPool> = Arc::clone(&recording) as _;+///+/// let reservation = MemoryConsumer::new("example").register(&pool);+/// reservation.try_grow(512)?;+/// reservation.shrink(512);+///+/// // The pool is back to empty, but the high-water mark is retained.+/// assert_eq!(pool.reserved(), 0);+/// assert_eq!(recording.peak_reserved(), 512);+///+/// // The recorder can also be recovered from the pool it was installed as.+/// assert_eq!(PeakRecordingPool::from_pool(&*pool).unwrap().peak_reserved(), 512);+/// # Ok::<(), datafusion_common::DataFusionError>(())+/// ```+pub struct PeakRecordingPool {+ inner: Arc<dyn MemoryPool>,+ /// Running total of everything granted through this wrapper, kept so the+ /// peak can be maintained without asking `inner` for its total.+ reserved: AtomicUsize,+ /// High-water mark since the last [`PeakRecordingPool::reset_peak`].+ peak: AtomicUsize,+ /// High-water mark since this pool was created. Never reset.+ max: AtomicUsize,+}++impl PeakRecordingPool {+ /// Wrap `inner`, recording its peak reservation from here on.+ ///+ /// `inner` is expected to be empty: the running total starts at zero, so+ /// anything reserved before wrapping is not counted.+ pub fn new(inner: Arc<dyn MemoryPool>) -> Self {+ Self {+ inner,+ reserved: AtomicUsize::new(0),+ peak: AtomicUsize::new(0),+ max: AtomicUsize::new(0),+ }+ }++ /// The recorder installed as `pool`, if there is one.+ ///+ /// Returns `None` whenever a benchmark runs without a memory limit, since+ /// [`CommonOpt::runtime_env_builder`] only installs the wrapper alongside a+ /// pool it has a limit for.+ ///+ /// [`CommonOpt::runtime_env_builder`]: super::CommonOpt::runtime_env_builder+ pub fn from_pool(pool: &dyn MemoryPool) -> Option<&Self> {+ pool.downcast_ref::<Self>()+ }++ /// Peak reservation, in bytes, since the last [`Self::reset_peak`].+ pub fn peak_reserved(&self) -> usize {+ self.peak.load(Ordering::Relaxed)+ }++ /// Peak reservation, in bytes, since this pool was created.+ ///+ /// Unlike [`Self::peak_reserved`] this is never reset, so it reports the+ /// peak across every query that shared this pool.+ pub fn max_reserved(&self) -> usize {+ self.max.load(Ordering::Relaxed)+ }++ /// Reset the value returned by [`Self::peak_reserved`] to what is reserved+ /// right now, so the next reading covers only what follows.+ ///+ /// [`BenchmarkRun::start_new_case`] calls this, giving each benchmark query+ /// its own reading. Anything still held when a query starts — data the+ /// benchmark loaded up front, say — stays in the reading, since the query+ /// runs with those bytes reserved.+ ///+ /// [`BenchmarkRun::start_new_case`]: super::BenchmarkRun::start_new_case+ pub fn reset_peak(&self) {+ self.peak+ .store(self.reserved.load(Ordering::Relaxed), Ordering::Relaxed);+ }++ /// Add `additional` granted bytes to the running total and publish it to+ /// both high-water marks.+ ///+ /// Accumulating deltas rather than reading [`MemoryPool::reserved`] keeps+ /// the wrapped pool's own bookkeeping off this path: `FairSpillPool` takes+ /// its state lock to answer `reserved()`, which would double the lock+ /// traffic of every accounted allocation in the benchmark being measured.+ /// The total stays exact because the trait grants exactly what is asked+ /// for — `grow` is infallible and `try_grow` either grants `additional` or+ /// returns an error, leaving the reservation untouched.+ fn record(&self, additional: usize) {+ let reserved =+ self.reserved.fetch_add(additional, Ordering::Relaxed) + additional;+ self.peak.fetch_max(reserved, Ordering::Relaxed);+ self.max.fetch_max(reserved, Ordering::Relaxed);+ }+}++impl Debug for PeakRecordingPool {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("PeakRecordingPool")+ .field("inner", &self.inner)+ .field("peak", &self.peak_reserved())+ .field("max", &self.max_reserved())+ .finish()+ }+}++impl Display for PeakRecordingPool {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ // Deferring to the wrapped pool keeps `SHOW ALL`-style output and error+ // messages identical to running without the wrapper.+ Display::fmt(&self.inner, f)+ }+}++impl MemoryPool for PeakRecordingPool {+ fn name(&self) -> &str {+ self.inner.name()+ }++ fn register(&self, consumer: &MemoryConsumer) {+ self.inner.register(consumer);+ }++ fn unregister(&self, consumer: &MemoryConsumer) {+ self.inner.unregister(consumer);+ }++ fn grow(&self, reservation: &MemoryReservation, additional: usize) {+ self.inner.grow(reservation, additional);+ self.record(additional);+ }++ fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {+ self.inner.shrink(reservation, shrink);+ self.reserved.fetch_sub(shrink, Ordering::Relaxed);+ }++ fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> {+ self.inner.try_grow(reservation, additional)?;+ self.record(additional);+ Ok(())+ }++ fn reserved(&self) -> usize {+ self.inner.reserved()+ }++ fn memory_limit(&self) -> MemoryLimit {+ self.inner.memory_limit()+ }+}++#[cfg(test)]+mod tests {+ use datafusion::execution::memory_pool::GreedyMemoryPool;++ use super::*;++ /// A recording pool over a `GreedyMemoryPool`, returned both as the+ /// recorder (to read the marks) and as the pool reservations register with.+ fn pool(limit: usize) -> (Arc<PeakRecordingPool>, Arc<dyn MemoryPool>) {+ let recording = Arc::new(PeakRecordingPool::new(Arc::new(+ GreedyMemoryPool::new(limit),+ )));+ let pool = Arc::clone(&recording) as Arc<dyn MemoryPool>;+ (recording, pool)+ }++ #[test]+ fn records_high_water_mark_across_reservations() {+ let (recording, pool) = pool(1024);++ let a = MemoryConsumer::new("a").register(&pool);+ let b = MemoryConsumer::new("b").register(&pool);++ a.try_grow(300).unwrap();+ b.try_grow(400).unwrap();+ // Peak of the sum, not the largest single reservation.+ assert_eq!(recording.peak_reserved(), 700);++ a.shrink(300);+ b.try_grow(100).unwrap();++ // Falling back below the peak leaves it untouched, and the later growth+ // does not reach it.+ assert_eq!(pool.reserved(), 500);+ assert_eq!(recording.peak_reserved(), 700);+ }++ #[test]+ fn failed_growth_does_not_move_the_peak() {+ let (recording, pool) = pool(1024);++ let reservation = MemoryConsumer::new("a").register(&pool);+ reservation.try_grow(600).unwrap();+ reservation+ .try_grow(600)+ .expect_err("should exceed the 1024 byte pool");++ assert_eq!(recording.peak_reserved(), 600);+ }++ #[test]+ fn reset_clears_the_window_but_not_the_run_maximum() {+ let (recording, pool) = pool(1024);++ let reservation = MemoryConsumer::new("a").register(&pool);+ reservation.try_grow(800).unwrap();+ reservation.shrink(800);++ recording.reset_peak();+ assert_eq!(recording.peak_reserved(), 0);+ assert_eq!(recording.max_reserved(), 800);++ reservation.try_grow(100).unwrap();+ assert_eq!(recording.peak_reserved(), 100);+ assert_eq!(recording.max_reserved(), 800);+ }++ #[test]+ fn reset_keeps_what_is_still_reserved() {+ let (recording, pool) = pool(1024);++ // Something a benchmark loaded up front and holds across queries.+ let held = MemoryConsumer::new("held").register(&pool);+ held.try_grow(300).unwrap();++ recording.reset_peak();+ assert_eq!(recording.peak_reserved(), 300);++ let query = MemoryConsumer::new("query").register(&pool);+ query.try_grow(200).unwrap();+ assert_eq!(recording.peak_reserved(), 500);+ }++ #[test]+ fn marks_are_per_instance() {+ let (one, one_pool) = pool(1024);+ let (two, _two_pool) = pool(1024);++ MemoryConsumer::new("a")+ .register(&one_pool)+ .try_grow(512)+ .unwrap();++ assert_eq!(one.peak_reserved(), 512);+ assert_eq!(two.peak_reserved(), 0);+ }++ #[test]+ fn is_recoverable_from_the_pool_it_is_installed_as() {+ let (recording, pool) = pool(1024);++ MemoryConsumer::new("a")+ .register(&pool)+ .try_grow(512)+ .unwrap();++ let found = PeakRecordingPool::from_pool(&*pool).expect("recorder installed");+ assert_eq!(found.peak_reserved(), recording.peak_reserved());++ // A pool with no recorder in front of it reports nothing.+ let plain: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1024));+ assert!(PeakRecordingPool::from_pool(&*plain).is_none());+ }++ #[test]+ fn delegates_limit_and_name_to_the_wrapped_pool() {+ let inner: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(4096));+ let wrapped = PeakRecordingPool::new(Arc::clone(&inner));++ assert_eq!(wrapped.name(), inner.name());+ assert_eq!(wrapped.to_string(), inner.to_string());+ assert!(matches!(wrapped.memory_limit(), MemoryLimit::Finite(4096)));+ }++ /// Arrow-side reservations reach the recorder too.+ ///+ /// [`ArrowMemoryPool`] implements Arrow's `MemoryPool` by growing a+ /// DataFusion [`MemoryReservation`] against the pool it wraps, so a buffer+ /// claimed through it lands in `grow` here. Nothing in DataFusion claims+ /// buffers yet (see apache/datafusion#22898), but when something does, the+ /// bytes show up in this peak without further changes — as long as the+ /// adapter is built from the `RuntimeEnv`'s pool, which is the wrapped one.+ /// This test pins that.+ #[test]+ fn records_reservations_arriving_through_the_arrow_adapter() {+ use arrow_buffer::MemoryPool as ArrowMemoryPoolTrait;+ use datafusion_execution::memory_pool::arrow::ArrowMemoryPool;++ let (recording, pool) = pool(4096);++ let arrow_pool =+ ArrowMemoryPool::new(Arc::clone(&pool), MemoryConsumer::new("arrow"));+ let reservation = arrow_pool.reserve(1024);++ // The Arrow-side reservation is visible as DataFusion pool usage...+ assert_eq!(pool.reserved(), 1024);+ assert_eq!(recording.peak_reserved(), 1024);++ // ...and dropping it releases the bytes while the peak is retained.+ drop(reservation);+ assert_eq!(pool.reserved(), 0);+ assert_eq!(recording.peak_reserved(), 1024);+ }+}diff --git a/benchmarks/src/util/mod.rs b/benchmarks/src/util/mod.rsindex 6dc11c0f42..43855ea468 100644--- a/benchmarks/src/util/mod.rs+++ b/benchmarks/src/util/mod.rs@@ -18,9 +18,11 @@//! Shared benchmark utilitiespub mod latency_object_store;mod memory;+mod memory_pool;mod options;mod run;pub use memory::print_memory_stats;+pub use memory_pool::PeakRecordingPool;pub use options::CommonOpt;pub use run::{BenchQuery, BenchmarkRun, QueryResult};diff --git a/benchmarks/src/util/options.rs b/benchmarks/src/util/options.rsindex a3e6d2a4c5..c744d0bf31 100644--- a/benchmarks/src/util/options.rs+++ b/benchmarks/src/util/options.rs@@ -30,7 +30,7 @@ use datafusion::{use datafusion_common::{DataFusionError, Result};use object_store::local::LocalFileSystem;-use super::latency_object_store::LatencyObjectStore;+use super::{latency_object_store::LatencyObjectStore, memory_pool::PeakRecordingPool};// Common benchmark options (don't use doc comments otherwise this doc// shows up in help files)@@ -125,6 +125,9 @@ impl CommonOpt {)));}};+ // Record the peak reservation so benchmarks can report it next to+ // peak RSS. Purely observational: every call is delegated.+ let pool: Arc<dyn MemoryPool> = Arc::new(PeakRecordingPool::new(pool));rt_builder = rt_builder.with_memory_pool(pool).with_disk_manager_builder(DiskManagerBuilder::default());diff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rsindex df17674e62..6c63ceec64 100644--- a/benchmarks/src/util/run.rs+++ b/benchmarks/src/util/run.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+use super::memory_pool::PeakRecordingPool;+use datafusion::execution::memory_pool::MemoryPool;use datafusion::{DATAFUSION_VERSION, error::Result};use datafusion_common::utils::get_available_parallelism;use serde::{Serialize, Serializer};@@ -22,6 +24,7 @@ use serde_json::Value;use std::{collections::HashMap,path::Path,+ sync::Arc,time::{Duration, SystemTime},};@@ -91,6 +94,16 @@ pub struct BenchQuery {#[serde(serialize_with = "serialize_start_time")]start_time: SystemTime,success: bool,+ /// Peak [`MemoryPool`] reservation observed while running this query, in+ /// bytes. Recorded for failed queries too, since a query that ran out of+ /// memory is one whose peak is worth seeing.+ ///+ /// `None` (and omitted from the JSON) only when the benchmark ran without a+ /// memory limit, since there is then no pool to record.+ ///+ /// [`MemoryPool`]: datafusion::execution::memory_pool::MemoryPool+ #[serde(skip_serializing_if = "Option::is_none")]+ pool_peak_bytes: Option<usize>,}/// Internal representation of a single benchmark query iteration result.pub struct QueryResult {@@ -102,6 +115,10 @@ pub struct BenchmarkRun {context: RunContext,queries: Vec<BenchQuery>,current_case: Option<usize>,+ /// The pool queries run against, when one was handed over with+ /// [`BenchmarkRun::set_memory_pool`]. Only read through+ /// [`BenchmarkRun::peak_recorder`].+ memory_pool: Option<Arc<dyn MemoryPool>>,}impl Default for BenchmarkRun {@@ -117,15 +134,44 @@ impl BenchmarkRun {context: RunContext::new(),queries: vec![],current_case: None,+ memory_pool: None,}}++ /// Report the peak reservation of `memory_pool` alongside each query.+ ///+ /// Call this with the pool of the [`RuntimeEnv`] the queries run against.+ /// Has no effect unless a [`PeakRecordingPool`] is installed, which+ /// [`CommonOpt::runtime_env_builder`] does whenever a memory limit is+ /// configured; without one `pool_peak_bytes` is omitted from the results.+ ///+ /// Benchmarks that build a runtime per query should call this each time, so+ /// each query reports against the pool it actually ran on.+ ///+ /// [`RuntimeEnv`]: datafusion::execution::runtime_env::RuntimeEnv+ /// [`CommonOpt::runtime_env_builder`]: super::CommonOpt::runtime_env_builder+ pub fn set_memory_pool(&mut self, memory_pool: &Arc<dyn MemoryPool>) {+ self.memory_pool = Some(Arc::clone(memory_pool));+ }++ /// The recorder in front of the pool set by [`Self::set_memory_pool`].+ fn peak_recorder(&self) -> Option<&PeakRecordingPool> {+ PeakRecordingPool::from_pool(self.memory_pool.as_deref()?)+ }+/// begin a new case. iterations added after this will be included in the new casepub fn start_new_case(&mut self, id: &str) {+ // Give this query its own memory pool reading rather than inheriting+ // the high-water mark of the queries that ran before it.+ if let Some(recorder) = self.peak_recorder() {+ recorder.reset_peak();+ }self.queries.push(BenchQuery {query: id.to_owned(),iterations: vec![],start_time: SystemTime::now(),success: true,+ pool_peak_bytes: None,});if let Some(c) = self.current_case.as_mut() {*c += 1;@@ -135,10 +181,14 @@ impl BenchmarkRun {}/// Write a new iteration to the current casepub fn write_iter(&mut self, elapsed: Duration, row_count: usize) {+ // The peak is not reset between iterations, so this ends up holding the+ // largest reservation seen across all of them.+ let pool_peak_bytes = self.peak_recorder().map(PeakRecordingPool::peak_reserved);if let Some(idx) = self.current_case {self.queries[idx].iterations- .push(QueryIter { elapsed, row_count })+ .push(QueryIter { elapsed, row_count });+ self.queries[idx].pool_peak_bytes = pool_peak_bytes;} else {panic!("no cases existed yet");}@@ -159,8 +209,12 @@ impl BenchmarkRun {/// Mark current querypub fn mark_failed(&mut self) {+ // A query that failed under a memory limit wrote no iteration, so this+ // is the only chance to record what it had reserved when it gave up.+ let pool_peak_bytes = self.peak_recorder().map(PeakRecordingPool::peak_reserved);if let Some(idx) = self.current_case {self.queries[idx].success = false;+ self.queries[idx].pool_peak_bytes = pool_peak_bytes;} else {unreachable!("Cannot mark failure: no current case");}@@ -182,3 +236,87 @@ impl BenchmarkRun {Ok(())}}++#[cfg(test)]+mod tests {+ use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryConsumer};++ use super::*;++ fn recording_pool(limit: usize) -> Arc<dyn MemoryPool> {+ Arc::new(PeakRecordingPool::new(Arc::new(GreedyMemoryPool::new(+ limit,+ ))))+ }++ #[test]+ fn each_case_reports_its_own_peak() {+ let pool = recording_pool(1024);+ let mut run = BenchmarkRun::new();+ run.set_memory_pool(&pool);++ run.start_new_case("q1");+ let reservation = MemoryConsumer::new("q1").register(&pool);+ reservation.try_grow(600).unwrap();+ run.write_iter(Duration::from_millis(1), 1);+ drop(reservation);++ // The second case must not inherit the first case's high-water mark.+ run.start_new_case("q2");+ let reservation = MemoryConsumer::new("q2").register(&pool);+ reservation.try_grow(100).unwrap();+ run.write_iter(Duration::from_millis(1), 1);++ assert_eq!(run.queries[0].pool_peak_bytes, Some(600));+ assert_eq!(run.queries[1].pool_peak_bytes, Some(100));+ }++ #[test]+ fn a_later_pool_replaces_an_earlier_one() {+ let first = recording_pool(1024);+ let mut run = BenchmarkRun::new();+ run.set_memory_pool(&first);+ MemoryConsumer::new("q1")+ .register(&first)+ .try_grow(600)+ .unwrap();++ // Benchmarks that build a runtime per query hand over the new pool+ // before the next case; the reading follows it.+ let second = recording_pool(1024);+ run.set_memory_pool(&second);+ run.start_new_case("q2");+ MemoryConsumer::new("q2")+ .register(&second)+ .try_grow(100)+ .unwrap();+ run.write_iter(Duration::from_millis(1), 1);++ assert_eq!(run.queries[0].pool_peak_bytes, Some(100));+ }++ #[test]+ fn a_failed_query_still_reports_its_peak() {+ let pool = recording_pool(1024);+ let mut run = BenchmarkRun::new();+ run.set_memory_pool(&pool);++ run.start_new_case("q1");+ let reservation = MemoryConsumer::new("q1").register(&pool);+ reservation.try_grow(600).unwrap();+ // No `write_iter`: the query failed before completing an iteration.+ run.mark_failed();++ assert_eq!(run.queries[0].pool_peak_bytes, Some(600));+ }++ #[test]+ fn the_peak_is_omitted_without_a_recording_pool() {+ let mut run = BenchmarkRun::new();+ run.start_new_case("q1");+ run.write_iter(Duration::from_millis(1), 1);++ assert_eq!(run.queries[0].pool_peak_bytes, None);+ assert!(!run.to_json().contains("pool_peak_bytes"));+ }+}