Skip to content

Apache DataFusion SQL Query Engine

bench: report per-query memory pool peaks from the Criterion SQL harness (#25644)

Adrian Garcia Badaracco7570366fd929
diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh
index 71953d75db..66b6dcd1b1 100755
--- a/benchmarks/bench.sh
+++ b/benchmarks/bench.sh
@@ -820,6 +820,17 @@ data_tpcds() {
echo "Done."
}
+# Path of the results JSON the Criterion SQL harness ($SQL_CARGO_COMMAND)
+# writes, passed to it as BENCH_RESULTS_FILE. It has the same format as the
+# dfbench results files and holds each query's peak memory pool reservation
+# (recorded only under a memory limit). It has no timings: Criterion keeps
+# those under target/criterion. It goes in a subdirectory so that `compare`,
+# which reads ${RESULTS_DIR}/*.json as timing results, does not read it.
+sql_results_file() {
+ mkdir -p "${RESULTS_DIR}/criterion"
+ echo "${RESULTS_DIR}/criterion/$1.json"
+}
+
# Runs the tpch benchmark
run_tpch() {
SCALE_FACTOR=$1
@@ -831,6 +842,7 @@ run_tpch() {
echo "Running tpch benchmark..."
debug_run env BENCH_NAME=tpch \
+ BENCH_RESULTS_FILE="$(sql_results_file tpch_sf${SCALE_FACTOR}_${FORMAT})" \
BENCH_SIZE="${SCALE_FACTOR}" \
DATA_DIR="${DATA_DIR}" \
PREFER_HASH_JOIN="${PREFER_HASH_JOIN}" \
@@ -891,6 +903,7 @@ data_wide_schema() {
run_wide_schema() {
echo "Running wide_schema benchmark (wide subgroup)..."
debug_run env BENCH_NAME=wide_schema BENCH_SUBGROUP=wide \
+ BENCH_RESULTS_FILE="$(sql_results_file wide_schema_wide)" \
DATA_DIR="${DATA_DIR}" \
SIMULATE_LATENCY="${SIMULATE_LATENCY}" \
${QUERY:+BENCH_QUERY="${QUERY}"} \
@@ -898,6 +911,7 @@ run_wide_schema() {
echo "Running wide_schema benchmark (narrow baseline subgroup)..."
debug_run env BENCH_NAME=wide_schema BENCH_SUBGROUP=narrow \
+ BENCH_RESULTS_FILE="$(sql_results_file wide_schema_narrow)" \
DATA_DIR="${DATA_DIR}" \
SIMULATE_LATENCY="${SIMULATE_LATENCY}" \
${QUERY:+BENCH_QUERY="${QUERY}"} \
@@ -911,6 +925,7 @@ run_push_down_topk() {
echo "Running push_down_topk benchmark..."
debug_run env BENCH_NAME=push_down_topk \
+ BENCH_RESULTS_FILE="$(sql_results_file push_down_topk)" \
BENCH_SIZE="1" \
DATA_DIR="${DATA_DIR}" \
SIMULATE_LATENCY="${SIMULATE_LATENCY}" \
@@ -942,6 +957,7 @@ run_push_down_topk() {
run_predicate_eval() {
echo "Running predicate_eval benchmark (subgroup=${BENCH_SUBGROUP:-all}, rows=${PRED_ROWS:-1000000})..."
debug_run env BENCH_NAME=predicate_eval \
+ BENCH_RESULTS_FILE="$(sql_results_file predicate_eval)" \
${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \
PRED_ROWS="${PRED_ROWS:-1000000}" \
${PRED_FILL:+PRED_FILL="${PRED_FILL}"} \
@@ -963,6 +979,7 @@ run_predicate_eval() {
run_parquet_row_filter_skip() {
echo "Running parquet_row_filter_skip benchmark (subgroup=${BENCH_SUBGROUP:-all}, rows=${PRED_ROWS:-10000000}, rg_size=${RG_SIZE:-1000000})..."
debug_run env BENCH_NAME=parquet_row_filter_skip \
+ BENCH_RESULTS_FILE="$(sql_results_file parquet_row_filter_skip)" \
${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \
PRED_ROWS="${PRED_ROWS:-10000000}" \
RG_SIZE="${RG_SIZE:-1000000}" \
@@ -985,6 +1002,7 @@ run_parquet_row_filter_skip() {
run_null_aware_join() {
echo "Running null_aware_join benchmark (rows=${NAJ_ROWS:-10000}, large_rows=${NAJ_LARGE_ROWS:-1000000})..."
debug_run env BENCH_NAME=null_aware_join \
+ BENCH_RESULTS_FILE="$(sql_results_file null_aware_join)" \
NAJ_ROWS="${NAJ_ROWS:-10000}" \
NAJ_LARGE_ROWS="${NAJ_LARGE_ROWS:-1000000}" \
${QUERY:+BENCH_QUERY="${QUERY}"} \
@@ -1004,6 +1022,7 @@ run_null_aware_join() {
run_projection_subquery() {
echo "Running projection_subquery benchmark (rows=${PSQ_ROWS:-30000})..."
debug_run env BENCH_NAME=projection_subquery \
+ BENCH_RESULTS_FILE="$(sql_results_file projection_subquery)" \
PSQ_ROWS="${PSQ_ROWS:-30000}" \
${QUERY:+BENCH_QUERY="${QUERY}"} \
bash -c "$SQL_CARGO_COMMAND"
@@ -1018,6 +1037,7 @@ run_projection_subquery() {
run_spill_views() {
echo "Running spill_views benchmark (subgroup=${BENCH_SUBGROUP:-all})..."
debug_run env BENCH_NAME=spill_views \
+ BENCH_RESULTS_FILE="$(sql_results_file spill_views)" \
${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \
${QUERY:+BENCH_QUERY="${QUERY}"} \
bash -c "$SQL_CARGO_COMMAND"
@@ -1033,6 +1053,7 @@ run_tpch_mem() {
echo "Running tpch_mem benchmark..."
debug_run env BENCH_NAME=tpch \
+ BENCH_RESULTS_FILE="$(sql_results_file tpch_mem_sf${SCALE_FACTOR})" \
BENCH_SIZE="${SCALE_FACTOR}" \
DATA_DIR="${DATA_DIR}" \
TPCH_FILE_TYPE="mem" \
@@ -1438,6 +1459,7 @@ run_h2o_window_sorted() {
FILE_TYPE=${2:-"csv"}
echo "Running h2o window_sorted benchmark (size=${SIZE}, source format=${FILE_TYPE})..."
debug_run env BENCH_NAME=h2o \
+ BENCH_RESULTS_FILE="$(sql_results_file h2o_window_sorted_${SIZE}_${FILE_TYPE})" \
BENCH_SUBGROUP=window_sorted \
H2O_BENCH_SIZE="${SIZE}" \
H2O_FILE_TYPE="${FILE_TYPE}" \
@@ -1813,6 +1835,7 @@ run_dict() {
run_array_agg_distinct() {
echo "Running array_agg_distinct benchmark..."
debug_run env BENCH_NAME=array_agg_distinct \
+ BENCH_RESULTS_FILE="$(sql_results_file array_agg_distinct)" \
${QUERY:+BENCH_QUERY="${QUERY}"} \
bash -c "$SQL_CARGO_COMMAND"
}
diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.md
index 340c05657e..8047c03092 100644
--- a/benchmarks/sql_benchmarks/README.md
+++ b/benchmarks/sql_benchmarks/README.md
@@ -148,6 +148,7 @@ The SQL benchmarking tool uses the following environment variables:
| BENCH_QUERY | A query number to run. |
| BENCH_PERSIST_RESULTS | true/false to persist benchmark results. Results will be persisted in csv format so be cognizant of the size of the results. |
| BENCH_VALIDATE | true/false to validate benchmark results against persisted results or result_query's. If both `BENCH_PERSIST_RESULTS` and `BENCH_VALIDATE` are true, persist mode runs and validation is skipped. |
+| BENCH_RESULTS_FILE | Criterion runs only: write a results JSON in the `dfbench` `-o` format to this path. Each query records its peak memory pool reservation (`pool_peak_bytes`, only under a memory limit, including one set with `SET datafusion.runtime.memory_limit`) and no timings, which stay with Criterion. `bench.sh` sets it to `results/<name>/criterion/<file>.json`. |
| DATA_DIR | Root directory for benchmark data loaded by SQL benchmark files. When unset, uses `data` (relative to the benchmarks/ directory). |
| SIMULATE_LATENCY | Simulate object store latency to mimic remote storage (e.g. S3). Adds random latency in the range 20-200ms to each object store operation. |
| MEM_POOL_TYPE | The memory pool type to use, should be one of "fair" or "greedy". |
diff --git a/benchmarks/src/sql_benchmark_runner.rs b/benchmarks/src/sql_benchmark_runner.rs
index b579c7aea5..baeb34ec3b 100644
--- a/benchmarks/src/sql_benchmark_runner.rs
+++ b/benchmarks/src/sql_benchmark_runner.rs
@@ -19,10 +19,13 @@
//! SQL benchmark harness.
use crate::sql_benchmark::SqlBenchmark;
-use crate::util::{CommonOpt, print_memory_stats};
+use crate::util::{BenchmarkRun, CommonOpt, print_memory_stats};
use clap::Parser;
use criterion::{Criterion, SamplingMode};
use datafusion::error::Result;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::memory_pool::{MemoryLimit, MemoryPool, PeakRecordingPool};
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::prelude::SessionContext;
use datafusion_common::{DataFusionError, exec_datafusion_err};
use std::any::Any;
@@ -31,6 +34,7 @@ use std::fmt::Write as _;
use std::fs;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::path::{Path, PathBuf};
+use std::sync::Arc;
use tokio::runtime::Runtime;
const CRITERION_MAX_DIRECTORY_NAME_LEN: usize = 64;
@@ -86,6 +90,12 @@ struct CriterionHarnessEnv {
#[arg(env = "BENCH_NAMESPACE")]
criterion_namespace: Option<String>,
+
+ /// Write each case's peak memory pool reservation to this path, in the
+ /// results JSON format `dfbench -o` writes. Timings stay with Criterion,
+ /// so every case's `iterations` list is empty.
+ #[arg(env = "BENCH_RESULTS_FILE", long = "results-file")]
+ results_file: Option<PathBuf>,
}
/// Builds the direct Criterion harness configuration from its `BENCH_*`
@@ -103,7 +113,7 @@ pub fn criterion_harness_config_from_env() -> (SqlRunConfig, Option<String>) {
query_filename: None,
persist_results: args.persist_results,
validate_results: args.validate,
- output: None,
+ output: args.results_file,
};
(config, args.criterion_namespace)
@@ -147,16 +157,40 @@ pub fn run_criterion_benchmarks_impl_with_namespace(
.push((criterion_group_name(&group_name, namespace)?, benchmarks));
}
+ let mut results = BenchmarkRun::new();
+ let outcome =
+ run_criterion_groups(&rt, named_benchmarks, config, criterion, &mut results);
+ // Written on failure too: a case that ran out of memory is one whose peak
+ // is worth seeing.
+ results.maybe_write_json(config.output.as_ref())?;
+
+ outcome
+}
+
+fn run_criterion_groups(
+ rt: &Runtime,
+ named_benchmarks: Vec<(String, Vec<SqlBenchmark>)>,
+ config: &SqlRunConfig,
+ criterion: &mut Criterion,
+ results: &mut BenchmarkRun,
+) -> Result<()> {
for (group_name, benchmarks) in named_benchmarks {
- let mut group = criterion.benchmark_group(group_name);
+ let mut group = criterion.benchmark_group(group_name.clone());
group.sample_size(10);
group.sampling_mode(SamplingMode::Flat);
for mut benchmark in benchmarks {
let ctx = make_ctx(&config.common)?;
- let result =
- run_criterion_benchmark(&rt, &ctx, &mut benchmark, config, &mut group);
+ let result = run_criterion_benchmark(
+ rt,
+ &ctx,
+ &mut benchmark,
+ config,
+ &group_name,
+ &mut group,
+ results,
+ );
let cleanup_result = rt.block_on(benchmark.cleanup(&ctx));
finish_benchmark(result, cleanup_result)?;
@@ -206,18 +240,36 @@ fn criterion_group_name(group_name: &str, namespace: Option<&str>) -> Result<Str
}
/// Runs one benchmark case inside Criterion and converts benchmark panics to errors.
+///
+/// Adds a case to `results` holding the peak memory pool reservation across
+/// every execution Criterion makes of the query, warm-up included. The
+/// untimed `load`, `init` and `assert` steps run before the case starts, so
+/// they are not in the reading.
fn run_criterion_benchmark(
rt: &Runtime,
ctx: &SessionContext,
benchmark: &mut SqlBenchmark,
config: &SqlRunConfig,
+ group_name: &str,
group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>,
+ results: &mut BenchmarkRun,
) -> Result<()> {
rt.block_on(prepare_benchmark(ctx, benchmark, config))?;
let name = criterion_function_name(benchmark);
+ // The id Criterion (and `critcmp`) reports the case under.
+ let case_id = format!("{group_name}/{name}");
+ results.set_memory_pool(&ctx.runtime_env().memory_pool);
+
+ // Criterion does not call the closure for a case its filter excludes, so
+ // the case is only started, and the peak reset, once it really runs.
+ let mut started = false;
let result = catch_unwind(AssertUnwindSafe(|| {
group.bench_function(name.clone(), |b| {
+ if !started {
+ results.start_new_case(&case_id);
+ started = true;
+ }
b.iter(|| {
let _ = rt.block_on(async {
benchmark.run(ctx, false).await.unwrap_or_else(|err| {
@@ -230,10 +282,18 @@ fn run_criterion_benchmark(
match result {
Ok(()) => {
+ if started {
+ results.record_pool_peak();
+ }
print_memory_stats(&*ctx.runtime_env().memory_pool);
Ok(())
}
- Err(payload) => Err(panic_payload_to_error(payload.as_ref())),
+ Err(payload) => {
+ if started {
+ results.mark_failed();
+ }
+ Err(panic_payload_to_error(payload.as_ref()))
+ }
}
}
@@ -615,6 +675,7 @@ pub async fn prepare_benchmark(
config: &SqlRunConfig,
) -> Result<()> {
benchmark.initialize(ctx).await?;
+ record_pool_peak_after_init(ctx)?;
benchmark.assert(ctx).await?;
if config.persist_results {
@@ -627,6 +688,44 @@ pub async fn prepare_benchmark(
Ok(())
}
+/// Puts a [`PeakRecordingPool`] back in front of the session's memory pool
+/// when the benchmark's own SQL replaced it.
+///
+/// [`CommonOpt::runtime_env_builder`] installs the recorder when the harness
+/// has a memory limit. A suite that sets its own limit with
+/// `SET datafusion.runtime.memory_limit` (`spill_views` does this in its `init`
+/// script) makes the `SessionContext` build a new `RuntimeEnv` with a new pool,
+/// and the recorder is lost. Without this step such a suite reports no peak,
+/// with or without a harness-level limit.
+///
+/// Only a pool with a finite limit gets a recorder, the same rule as
+/// `runtime_env_builder`, so a run with no limit at all still reports nothing.
+fn record_pool_peak_after_init(ctx: &SessionContext) -> Result<()> {
+ let runtime = ctx.runtime_env();
+ let pool = &runtime.memory_pool;
+ if PeakRecordingPool::from_pool(pool.as_ref()).is_some()
+ || !matches!(pool.memory_limit(), MemoryLimit::Finite(_))
+ {
+ return Ok(());
+ }
+
+ let recorder: Arc<dyn MemoryPool> =
+ Arc::new(PeakRecordingPool::new(Arc::clone(pool)));
+ let runtime = RuntimeEnvBuilder::from_runtime_env(&runtime)
+ .with_memory_pool(recorder)
+ .build_arc()?;
+
+ // The same replacement `SET datafusion.runtime.*` makes, so everything
+ // registered on the session (tables, config) is kept.
+ let state = ctx.state_ref();
+ let mut state = state.write();
+ *state = SessionStateBuilder::from(state.clone())
+ .with_runtime_env(runtime)
+ .build();
+
+ Ok(())
+}
+
/// Ensures filtering selected at least one benchmark and emits targeted errors.
pub fn ensure_selection(
filter: &BenchmarkFilter,
@@ -708,6 +807,110 @@ mod tests {
path
}
+ fn common_opt(memory_limit: Option<usize>) -> CommonOpt {
+ CommonOpt {
+ iterations: 1,
+ partitions: None,
+ batch_size: None,
+ mem_pool_type: "fair".to_string(),
+ memory_limit,
+ sort_spill_reservation_bytes: None,
+ debug: false,
+ simulate_latency: false,
+ }
+ }
+
+ #[tokio::test]
+ async fn sql_memory_limit_keeps_the_peak_recorder() {
+ // The harness installs a recorder for its own limit; the SQL limit
+ // replaces that pool, and the recorder must follow it.
+ let ctx = make_ctx(&common_opt(Some(1024 * 1024 * 1024))).unwrap();
+ ctx.sql("SET datafusion.runtime.memory_limit = '100M'")
+ .await
+ .unwrap();
+ let pool = &ctx.runtime_env().memory_pool;
+ assert!(PeakRecordingPool::from_pool(pool.as_ref()).is_none());
+
+ record_pool_peak_after_init(&ctx).unwrap();
+
+ let pool = Arc::clone(&ctx.runtime_env().memory_pool);
+ assert!(PeakRecordingPool::from_pool(pool.as_ref()).is_some());
+ assert!(matches!(
+ pool.memory_limit(),
+ MemoryLimit::Finite(limit) if limit == 100 * 1024 * 1024
+ ));
+
+ // A query planned after the swap reserves through the recorder.
+ ctx.sql(
+ "SELECT value % 1000, count(*) FROM generate_series(1, 100000) GROUP BY 1",
+ )
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ let recorder = PeakRecordingPool::from_pool(pool.as_ref()).unwrap();
+ assert!(recorder.peak_reserved() > 0);
+
+ // A pool that already records is left alone.
+ record_pool_peak_after_init(&ctx).unwrap();
+ assert!(Arc::ptr_eq(&pool, &ctx.runtime_env().memory_pool));
+ }
+
+ #[tokio::test]
+ async fn no_memory_limit_gets_no_recorder() {
+ let ctx = make_ctx(&common_opt(None)).unwrap();
+
+ record_pool_peak_after_init(&ctx).unwrap();
+
+ let pool = &ctx.runtime_env().memory_pool;
+ assert!(PeakRecordingPool::from_pool(pool.as_ref()).is_none());
+ }
+
+ #[test]
+ fn criterion_harness_writes_pool_peak_per_case() {
+ let temp = tempfile::tempdir().unwrap();
+ // Like spill_views: the limit comes from SQL in `init`, not from the
+ // harness.
+ write_benchmark(
+ temp.path(),
+ "alpha/benchmarks/q01.benchmark",
+ "name Q01\n\ninit\nSET datafusion.runtime.memory_limit = '100M';\n\n\
+ run\nSELECT value % 1000, count(*) FROM generate_series(1, 100000) GROUP BY 1\n",
+ );
+ let results_file = temp.path().join("results.json");
+ let criterion_dir = tempfile::tempdir().unwrap();
+ let mut criterion = Criterion::default()
+ .warm_up_time(std::time::Duration::from_millis(1))
+ .measurement_time(std::time::Duration::from_millis(10))
+ .without_plots()
+ .output_directory(criterion_dir.path());
+ let config = SqlRunConfig {
+ common: common_opt(None),
+ filter: BenchmarkFilter {
+ name: Some("alpha".to_string()),
+ subgroup: None,
+ query: None,
+ },
+ replacements: HashMap::new(),
+ query_filename: None,
+ persist_results: false,
+ validate_results: false,
+ output: Some(results_file.clone()),
+ };
+
+ run_criterion_benchmarks_impl(temp.path(), &config, &mut criterion).unwrap();
+
+ let json: serde_json::Value =
+ serde_json::from_str(&fs::read_to_string(&results_file).unwrap()).unwrap();
+ let queries = json["queries"].as_array().unwrap();
+ assert_eq!(queries.len(), 1);
+ assert_eq!(queries[0]["query"], "alpha/Q01");
+ assert_eq!(queries[0]["iterations"], serde_json::json!([]));
+ let peak = queries[0]["pool_peak_bytes"].as_u64().unwrap();
+ assert!(peak > 0 && peak <= 100 * 1024 * 1024, "{peak}");
+ }
+
#[tokio::test]
async fn caller_replacements_reach_parser() {
let temp = tempfile::tempdir().unwrap();
diff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rs
index 829566d47c..130b902d70 100644
--- a/benchmarks/src/util/run.rs
+++ b/benchmarks/src/util/run.rs
@@ -193,6 +193,21 @@ impl BenchmarkRun {
}
}
+ /// Record the current case's peak pool reservation without adding an
+ /// iteration.
+ ///
+ /// For harnesses that time the query elsewhere: the Criterion SQL harness
+ /// leaves timings to Criterion and reports only the peak here, so the
+ /// case's `iterations` stay empty.
+ pub fn record_pool_peak(&mut self) {
+ let pool_peak_bytes = self.peak_recorder().map(PeakRecordingPool::peak_reserved);
+ if let Some(idx) = self.current_case {
+ self.queries[idx].pool_peak_bytes = pool_peak_bytes;
+ } else {
+ panic!("no cases existed yet");
+ }
+ }
+
/// Print the names of failed queries, if any
pub fn maybe_print_failures(&self) {
let failed_queries: Vec<&str> = self
@@ -309,6 +324,22 @@ mod tests {
assert_eq!(run.queries[0].pool_peak_bytes, Some(600));
}
+ #[test]
+ fn a_case_without_iterations_still_reports_its_peak() {
+ let pool = recording_pool(1024);
+ let mut run = BenchmarkRun::new();
+ run.set_memory_pool(&pool);
+
+ run.start_new_case("q1");
+ let reservation = MemoryConsumer::new("q1").register(&pool);
+ reservation.try_grow(600).unwrap();
+ drop(reservation);
+ run.record_pool_peak();
+
+ assert_eq!(run.queries[0].pool_peak_bytes, Some(600));
+ assert!(run.queries[0].iterations.is_empty());
+ }
+
#[test]
fn the_peak_is_omitted_without_a_recording_pool() {
let mut run = BenchmarkRun::new();