Apache DataFusion SQL Query Engine
bench: report per-query memory pool peaks from the Criterion SQL harness (#25644)
Adrian Garcia Badaracco7570366fd929
diff --git a/benchmarks/bench.sh b/benchmarks/bench.shindex 71953d75db..66b6dcd1b1 100755--- a/benchmarks/bench.sh+++ b/benchmarks/bench.sh@@ -820,6 +820,17 @@ data_tpcds() {echo "Done."}+# Path of the results JSON the Criterion SQL harness ($SQL_CARGO_COMMAND)+# writes, passed to it as BENCH_RESULTS_FILE. It has the same format as the+# dfbench results files and holds each query's peak memory pool reservation+# (recorded only under a memory limit). It has no timings: Criterion keeps+# those under target/criterion. It goes in a subdirectory so that `compare`,+# which reads ${RESULTS_DIR}/*.json as timing results, does not read it.+sql_results_file() {+ mkdir -p "${RESULTS_DIR}/criterion"+ echo "${RESULTS_DIR}/criterion/$1.json"+}+# Runs the tpch benchmarkrun_tpch() {SCALE_FACTOR=$1@@ -831,6 +842,7 @@ run_tpch() {echo "Running tpch benchmark..."debug_run env BENCH_NAME=tpch \+ BENCH_RESULTS_FILE="$(sql_results_file tpch_sf${SCALE_FACTOR}_${FORMAT})" \BENCH_SIZE="${SCALE_FACTOR}" \DATA_DIR="${DATA_DIR}" \PREFER_HASH_JOIN="${PREFER_HASH_JOIN}" \@@ -891,6 +903,7 @@ data_wide_schema() {run_wide_schema() {echo "Running wide_schema benchmark (wide subgroup)..."debug_run env BENCH_NAME=wide_schema BENCH_SUBGROUP=wide \+ BENCH_RESULTS_FILE="$(sql_results_file wide_schema_wide)" \DATA_DIR="${DATA_DIR}" \SIMULATE_LATENCY="${SIMULATE_LATENCY}" \${QUERY:+BENCH_QUERY="${QUERY}"} \@@ -898,6 +911,7 @@ run_wide_schema() {echo "Running wide_schema benchmark (narrow baseline subgroup)..."debug_run env BENCH_NAME=wide_schema BENCH_SUBGROUP=narrow \+ BENCH_RESULTS_FILE="$(sql_results_file wide_schema_narrow)" \DATA_DIR="${DATA_DIR}" \SIMULATE_LATENCY="${SIMULATE_LATENCY}" \${QUERY:+BENCH_QUERY="${QUERY}"} \@@ -911,6 +925,7 @@ run_push_down_topk() {echo "Running push_down_topk benchmark..."debug_run env BENCH_NAME=push_down_topk \+ BENCH_RESULTS_FILE="$(sql_results_file push_down_topk)" \BENCH_SIZE="1" \DATA_DIR="${DATA_DIR}" \SIMULATE_LATENCY="${SIMULATE_LATENCY}" \@@ -942,6 +957,7 @@ run_push_down_topk() {run_predicate_eval() {echo "Running predicate_eval benchmark (subgroup=${BENCH_SUBGROUP:-all}, rows=${PRED_ROWS:-1000000})..."debug_run env BENCH_NAME=predicate_eval \+ BENCH_RESULTS_FILE="$(sql_results_file predicate_eval)" \${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \PRED_ROWS="${PRED_ROWS:-1000000}" \${PRED_FILL:+PRED_FILL="${PRED_FILL}"} \@@ -963,6 +979,7 @@ run_predicate_eval() {run_parquet_row_filter_skip() {echo "Running parquet_row_filter_skip benchmark (subgroup=${BENCH_SUBGROUP:-all}, rows=${PRED_ROWS:-10000000}, rg_size=${RG_SIZE:-1000000})..."debug_run env BENCH_NAME=parquet_row_filter_skip \+ BENCH_RESULTS_FILE="$(sql_results_file parquet_row_filter_skip)" \${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \PRED_ROWS="${PRED_ROWS:-10000000}" \RG_SIZE="${RG_SIZE:-1000000}" \@@ -985,6 +1002,7 @@ run_parquet_row_filter_skip() {run_null_aware_join() {echo "Running null_aware_join benchmark (rows=${NAJ_ROWS:-10000}, large_rows=${NAJ_LARGE_ROWS:-1000000})..."debug_run env BENCH_NAME=null_aware_join \+ BENCH_RESULTS_FILE="$(sql_results_file null_aware_join)" \NAJ_ROWS="${NAJ_ROWS:-10000}" \NAJ_LARGE_ROWS="${NAJ_LARGE_ROWS:-1000000}" \${QUERY:+BENCH_QUERY="${QUERY}"} \@@ -1004,6 +1022,7 @@ run_null_aware_join() {run_projection_subquery() {echo "Running projection_subquery benchmark (rows=${PSQ_ROWS:-30000})..."debug_run env BENCH_NAME=projection_subquery \+ BENCH_RESULTS_FILE="$(sql_results_file projection_subquery)" \PSQ_ROWS="${PSQ_ROWS:-30000}" \${QUERY:+BENCH_QUERY="${QUERY}"} \bash -c "$SQL_CARGO_COMMAND"@@ -1018,6 +1037,7 @@ run_projection_subquery() {run_spill_views() {echo "Running spill_views benchmark (subgroup=${BENCH_SUBGROUP:-all})..."debug_run env BENCH_NAME=spill_views \+ BENCH_RESULTS_FILE="$(sql_results_file spill_views)" \${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \${QUERY:+BENCH_QUERY="${QUERY}"} \bash -c "$SQL_CARGO_COMMAND"@@ -1033,6 +1053,7 @@ run_tpch_mem() {echo "Running tpch_mem benchmark..."debug_run env BENCH_NAME=tpch \+ BENCH_RESULTS_FILE="$(sql_results_file tpch_mem_sf${SCALE_FACTOR})" \BENCH_SIZE="${SCALE_FACTOR}" \DATA_DIR="${DATA_DIR}" \TPCH_FILE_TYPE="mem" \@@ -1438,6 +1459,7 @@ run_h2o_window_sorted() {FILE_TYPE=${2:-"csv"}echo "Running h2o window_sorted benchmark (size=${SIZE}, source format=${FILE_TYPE})..."debug_run env BENCH_NAME=h2o \+ BENCH_RESULTS_FILE="$(sql_results_file h2o_window_sorted_${SIZE}_${FILE_TYPE})" \BENCH_SUBGROUP=window_sorted \H2O_BENCH_SIZE="${SIZE}" \H2O_FILE_TYPE="${FILE_TYPE}" \@@ -1813,6 +1835,7 @@ run_dict() {run_array_agg_distinct() {echo "Running array_agg_distinct benchmark..."debug_run env BENCH_NAME=array_agg_distinct \+ BENCH_RESULTS_FILE="$(sql_results_file array_agg_distinct)" \${QUERY:+BENCH_QUERY="${QUERY}"} \bash -c "$SQL_CARGO_COMMAND"}diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.mdindex 340c05657e..8047c03092 100644--- a/benchmarks/sql_benchmarks/README.md+++ b/benchmarks/sql_benchmarks/README.md@@ -148,6 +148,7 @@ The SQL benchmarking tool uses the following environment variables:| BENCH_QUERY | A query number to run. || BENCH_PERSIST_RESULTS | true/false to persist benchmark results. Results will be persisted in csv format so be cognizant of the size of the results. || BENCH_VALIDATE | true/false to validate benchmark results against persisted results or result_query's. If both `BENCH_PERSIST_RESULTS` and `BENCH_VALIDATE` are true, persist mode runs and validation is skipped. |+| BENCH_RESULTS_FILE | Criterion runs only: write a results JSON in the `dfbench` `-o` format to this path. Each query records its peak memory pool reservation (`pool_peak_bytes`, only under a memory limit, including one set with `SET datafusion.runtime.memory_limit`) and no timings, which stay with Criterion. `bench.sh` sets it to `results/<name>/criterion/<file>.json`. || DATA_DIR | Root directory for benchmark data loaded by SQL benchmark files. When unset, uses `data` (relative to the benchmarks/ directory). || SIMULATE_LATENCY | Simulate object store latency to mimic remote storage (e.g. S3). Adds random latency in the range 20-200ms to each object store operation. || MEM_POOL_TYPE | The memory pool type to use, should be one of "fair" or "greedy". |diff --git a/benchmarks/src/sql_benchmark_runner.rs b/benchmarks/src/sql_benchmark_runner.rsindex b579c7aea5..baeb34ec3b 100644--- a/benchmarks/src/sql_benchmark_runner.rs+++ b/benchmarks/src/sql_benchmark_runner.rs@@ -19,10 +19,13 @@//! SQL benchmark harness.use crate::sql_benchmark::SqlBenchmark;-use crate::util::{CommonOpt, print_memory_stats};+use crate::util::{BenchmarkRun, CommonOpt, print_memory_stats};use clap::Parser;use criterion::{Criterion, SamplingMode};use datafusion::error::Result;+use datafusion::execution::SessionStateBuilder;+use datafusion::execution::memory_pool::{MemoryLimit, MemoryPool, PeakRecordingPool};+use datafusion::execution::runtime_env::RuntimeEnvBuilder;use datafusion::prelude::SessionContext;use datafusion_common::{DataFusionError, exec_datafusion_err};use std::any::Any;@@ -31,6 +34,7 @@ use std::fmt::Write as _;use std::fs;use std::panic::{AssertUnwindSafe, catch_unwind};use std::path::{Path, PathBuf};+use std::sync::Arc;use tokio::runtime::Runtime;const CRITERION_MAX_DIRECTORY_NAME_LEN: usize = 64;@@ -86,6 +90,12 @@ struct CriterionHarnessEnv {#[arg(env = "BENCH_NAMESPACE")]criterion_namespace: Option<String>,++ /// Write each case's peak memory pool reservation to this path, in the+ /// results JSON format `dfbench -o` writes. Timings stay with Criterion,+ /// so every case's `iterations` list is empty.+ #[arg(env = "BENCH_RESULTS_FILE", long = "results-file")]+ results_file: Option<PathBuf>,}/// Builds the direct Criterion harness configuration from its `BENCH_*`@@ -103,7 +113,7 @@ pub fn criterion_harness_config_from_env() -> (SqlRunConfig, Option<String>) {query_filename: None,persist_results: args.persist_results,validate_results: args.validate,- output: None,+ output: args.results_file,};(config, args.criterion_namespace)@@ -147,16 +157,40 @@ pub fn run_criterion_benchmarks_impl_with_namespace(.push((criterion_group_name(&group_name, namespace)?, benchmarks));}+ let mut results = BenchmarkRun::new();+ let outcome =+ run_criterion_groups(&rt, named_benchmarks, config, criterion, &mut results);+ // Written on failure too: a case that ran out of memory is one whose peak+ // is worth seeing.+ results.maybe_write_json(config.output.as_ref())?;++ outcome+}++fn run_criterion_groups(+ rt: &Runtime,+ named_benchmarks: Vec<(String, Vec<SqlBenchmark>)>,+ config: &SqlRunConfig,+ criterion: &mut Criterion,+ results: &mut BenchmarkRun,+) -> Result<()> {for (group_name, benchmarks) in named_benchmarks {- let mut group = criterion.benchmark_group(group_name);+ let mut group = criterion.benchmark_group(group_name.clone());group.sample_size(10);group.sampling_mode(SamplingMode::Flat);for mut benchmark in benchmarks {let ctx = make_ctx(&config.common)?;- let result =- run_criterion_benchmark(&rt, &ctx, &mut benchmark, config, &mut group);+ let result = run_criterion_benchmark(+ rt,+ &ctx,+ &mut benchmark,+ config,+ &group_name,+ &mut group,+ results,+ );let cleanup_result = rt.block_on(benchmark.cleanup(&ctx));finish_benchmark(result, cleanup_result)?;@@ -206,18 +240,36 @@ fn criterion_group_name(group_name: &str, namespace: Option<&str>) -> Result<Str}/// Runs one benchmark case inside Criterion and converts benchmark panics to errors.+///+/// Adds a case to `results` holding the peak memory pool reservation across+/// every execution Criterion makes of the query, warm-up included. The+/// untimed `load`, `init` and `assert` steps run before the case starts, so+/// they are not in the reading.fn run_criterion_benchmark(rt: &Runtime,ctx: &SessionContext,benchmark: &mut SqlBenchmark,config: &SqlRunConfig,+ group_name: &str,group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>,+ results: &mut BenchmarkRun,) -> Result<()> {rt.block_on(prepare_benchmark(ctx, benchmark, config))?;let name = criterion_function_name(benchmark);+ // The id Criterion (and `critcmp`) reports the case under.+ let case_id = format!("{group_name}/{name}");+ results.set_memory_pool(&ctx.runtime_env().memory_pool);++ // Criterion does not call the closure for a case its filter excludes, so+ // the case is only started, and the peak reset, once it really runs.+ let mut started = false;let result = catch_unwind(AssertUnwindSafe(|| {group.bench_function(name.clone(), |b| {+ if !started {+ results.start_new_case(&case_id);+ started = true;+ }b.iter(|| {let _ = rt.block_on(async {benchmark.run(ctx, false).await.unwrap_or_else(|err| {@@ -230,10 +282,18 @@ fn run_criterion_benchmark(match result {Ok(()) => {+ if started {+ results.record_pool_peak();+ }print_memory_stats(&*ctx.runtime_env().memory_pool);Ok(())}- Err(payload) => Err(panic_payload_to_error(payload.as_ref())),+ Err(payload) => {+ if started {+ results.mark_failed();+ }+ Err(panic_payload_to_error(payload.as_ref()))+ }}}@@ -615,6 +675,7 @@ pub async fn prepare_benchmark(config: &SqlRunConfig,) -> Result<()> {benchmark.initialize(ctx).await?;+ record_pool_peak_after_init(ctx)?;benchmark.assert(ctx).await?;if config.persist_results {@@ -627,6 +688,44 @@ pub async fn prepare_benchmark(Ok(())}+/// Puts a [`PeakRecordingPool`] back in front of the session's memory pool+/// when the benchmark's own SQL replaced it.+///+/// [`CommonOpt::runtime_env_builder`] installs the recorder when the harness+/// has a memory limit. A suite that sets its own limit with+/// `SET datafusion.runtime.memory_limit` (`spill_views` does this in its `init`+/// script) makes the `SessionContext` build a new `RuntimeEnv` with a new pool,+/// and the recorder is lost. Without this step such a suite reports no peak,+/// with or without a harness-level limit.+///+/// Only a pool with a finite limit gets a recorder, the same rule as+/// `runtime_env_builder`, so a run with no limit at all still reports nothing.+fn record_pool_peak_after_init(ctx: &SessionContext) -> Result<()> {+ let runtime = ctx.runtime_env();+ let pool = &runtime.memory_pool;+ if PeakRecordingPool::from_pool(pool.as_ref()).is_some()+ || !matches!(pool.memory_limit(), MemoryLimit::Finite(_))+ {+ return Ok(());+ }++ let recorder: Arc<dyn MemoryPool> =+ Arc::new(PeakRecordingPool::new(Arc::clone(pool)));+ let runtime = RuntimeEnvBuilder::from_runtime_env(&runtime)+ .with_memory_pool(recorder)+ .build_arc()?;++ // The same replacement `SET datafusion.runtime.*` makes, so everything+ // registered on the session (tables, config) is kept.+ let state = ctx.state_ref();+ let mut state = state.write();+ *state = SessionStateBuilder::from(state.clone())+ .with_runtime_env(runtime)+ .build();++ Ok(())+}+/// Ensures filtering selected at least one benchmark and emits targeted errors.pub fn ensure_selection(filter: &BenchmarkFilter,@@ -708,6 +807,110 @@ mod tests {path}+ fn common_opt(memory_limit: Option<usize>) -> CommonOpt {+ CommonOpt {+ iterations: 1,+ partitions: None,+ batch_size: None,+ mem_pool_type: "fair".to_string(),+ memory_limit,+ sort_spill_reservation_bytes: None,+ debug: false,+ simulate_latency: false,+ }+ }++ #[tokio::test]+ async fn sql_memory_limit_keeps_the_peak_recorder() {+ // The harness installs a recorder for its own limit; the SQL limit+ // replaces that pool, and the recorder must follow it.+ let ctx = make_ctx(&common_opt(Some(1024 * 1024 * 1024))).unwrap();+ ctx.sql("SET datafusion.runtime.memory_limit = '100M'")+ .await+ .unwrap();+ let pool = &ctx.runtime_env().memory_pool;+ assert!(PeakRecordingPool::from_pool(pool.as_ref()).is_none());++ record_pool_peak_after_init(&ctx).unwrap();++ let pool = Arc::clone(&ctx.runtime_env().memory_pool);+ assert!(PeakRecordingPool::from_pool(pool.as_ref()).is_some());+ assert!(matches!(+ pool.memory_limit(),+ MemoryLimit::Finite(limit) if limit == 100 * 1024 * 1024+ ));++ // A query planned after the swap reserves through the recorder.+ ctx.sql(+ "SELECT value % 1000, count(*) FROM generate_series(1, 100000) GROUP BY 1",+ )+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ let recorder = PeakRecordingPool::from_pool(pool.as_ref()).unwrap();+ assert!(recorder.peak_reserved() > 0);++ // A pool that already records is left alone.+ record_pool_peak_after_init(&ctx).unwrap();+ assert!(Arc::ptr_eq(&pool, &ctx.runtime_env().memory_pool));+ }++ #[tokio::test]+ async fn no_memory_limit_gets_no_recorder() {+ let ctx = make_ctx(&common_opt(None)).unwrap();++ record_pool_peak_after_init(&ctx).unwrap();++ let pool = &ctx.runtime_env().memory_pool;+ assert!(PeakRecordingPool::from_pool(pool.as_ref()).is_none());+ }++ #[test]+ fn criterion_harness_writes_pool_peak_per_case() {+ let temp = tempfile::tempdir().unwrap();+ // Like spill_views: the limit comes from SQL in `init`, not from the+ // harness.+ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\ninit\nSET datafusion.runtime.memory_limit = '100M';\n\n\+ run\nSELECT value % 1000, count(*) FROM generate_series(1, 100000) GROUP BY 1\n",+ );+ let results_file = temp.path().join("results.json");+ let criterion_dir = tempfile::tempdir().unwrap();+ let mut criterion = Criterion::default()+ .warm_up_time(std::time::Duration::from_millis(1))+ .measurement_time(std::time::Duration::from_millis(10))+ .without_plots()+ .output_directory(criterion_dir.path());+ let config = SqlRunConfig {+ common: common_opt(None),+ filter: BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: None,+ query: None,+ },+ replacements: HashMap::new(),+ query_filename: None,+ persist_results: false,+ validate_results: false,+ output: Some(results_file.clone()),+ };++ run_criterion_benchmarks_impl(temp.path(), &config, &mut criterion).unwrap();++ let json: serde_json::Value =+ serde_json::from_str(&fs::read_to_string(&results_file).unwrap()).unwrap();+ let queries = json["queries"].as_array().unwrap();+ assert_eq!(queries.len(), 1);+ assert_eq!(queries[0]["query"], "alpha/Q01");+ assert_eq!(queries[0]["iterations"], serde_json::json!([]));+ let peak = queries[0]["pool_peak_bytes"].as_u64().unwrap();+ assert!(peak > 0 && peak <= 100 * 1024 * 1024, "{peak}");+ }+#[tokio::test]async fn caller_replacements_reach_parser() {let temp = tempfile::tempdir().unwrap();diff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rsindex 829566d47c..130b902d70 100644--- a/benchmarks/src/util/run.rs+++ b/benchmarks/src/util/run.rs@@ -193,6 +193,21 @@ impl BenchmarkRun {}}+ /// Record the current case's peak pool reservation without adding an+ /// iteration.+ ///+ /// For harnesses that time the query elsewhere: the Criterion SQL harness+ /// leaves timings to Criterion and reports only the peak here, so the+ /// case's `iterations` stay empty.+ pub fn record_pool_peak(&mut self) {+ let pool_peak_bytes = self.peak_recorder().map(PeakRecordingPool::peak_reserved);+ if let Some(idx) = self.current_case {+ self.queries[idx].pool_peak_bytes = pool_peak_bytes;+ } else {+ panic!("no cases existed yet");+ }+ }+/// Print the names of failed queries, if anypub fn maybe_print_failures(&self) {let failed_queries: Vec<&str> = self@@ -309,6 +324,22 @@ mod tests {assert_eq!(run.queries[0].pool_peak_bytes, Some(600));}+ #[test]+ fn a_case_without_iterations_still_reports_its_peak() {+ let pool = recording_pool(1024);+ let mut run = BenchmarkRun::new();+ run.set_memory_pool(&pool);++ run.start_new_case("q1");+ let reservation = MemoryConsumer::new("q1").register(&pool);+ reservation.try_grow(600).unwrap();+ drop(reservation);+ run.record_pool_peak();++ assert_eq!(run.queries[0].pool_peak_bytes, Some(600));+ assert!(run.queries[0].iterations.is_empty());+ }+#[test]fn the_peak_is_omitted_without_a_recording_pool() {let mut run = BenchmarkRun::new();