Skip to content

Apache DataFusion SQL Query Engine

Rich t kid/introduce dict benchmarks (#21860)

RIchard Baah6b27d2d55235
diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh
index 7a9a5fcdf3..29957f25e3 100755
--- a/benchmarks/bench.sh
+++ b/benchmarks/bench.sh
@@ -147,6 +147,7 @@ cancellation: How long cancelling a query takes
nlj: Benchmark for simple nested loop joins, testing various join scenarios
hj: Benchmark for simple hash joins, testing various join scenarios
smj: Benchmark for simple sort merge joins, testing various join scenarios
+dict: Benchmark for dictionary-encoded group-by scenarios
compile_profile: Compile and execute TPC-H across selected Cargo profiles, reporting timing and binary size
@@ -351,6 +352,10 @@ main() {
# smj uses range() function, no data generation needed
echo "SMJ benchmark does not require data generation"
;;
+ dict)
+ # dict generates in-memory data, no data generation needed
+ echo "DICT benchmark does not require data generation"
+ ;;
compile_profile)
data_tpch "1" "parquet"
;;
@@ -430,6 +435,7 @@ main() {
run_hj
run_tpcds
run_smj
+ run_dict
;;
tpch)
run_tpch "1" "parquet"
@@ -564,6 +570,9 @@ main() {
smj)
run_smj
;;
+ dict)
+ run_dict
+ ;;
compile_profile)
run_compile_profile "${PROFILE_ARGS[@]}"
;;
@@ -1460,6 +1469,14 @@ run_smj() {
debug_run $CARGO_COMMAND --bin dfbench -- smj --iterations 5 -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
}
+# Runs the dict benchmark
+run_dict() {
+ RESULTS_FILE="${RESULTS_DIR}/dict.json"
+ echo "RESULTS_FILE: ${RESULTS_FILE}"
+ echo "Running dict benchmark..."
+ debug_run $CARGO_COMMAND --bin dfbench -- dict --iterations 5 -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
+}
+
compare_benchmarks() {
BASE_RESULTS_DIR="${SCRIPT_DIR}/results"
diff --git a/benchmarks/src/bin/dfbench.rs b/benchmarks/src/bin/dfbench.rs
index cef73ad683..50dd99368b 100644
--- a/benchmarks/src/bin/dfbench.rs
+++ b/benchmarks/src/bin/dfbench.rs
@@ -17,6 +17,7 @@
//! DataFusion benchmark runner
use datafusion::error::Result;
+use datafusion_benchmarks::sort_pushdown;
use clap::{Parser, Subcommand};
@@ -31,8 +32,7 @@ static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
use datafusion_benchmarks::{
- cancellation, clickbench, h2o, hj, imdb, nlj, smj, sort_pushdown, sort_tpch, tpcds,
- tpch,
+ cancellation, clickbench, dict, h2o, hj, imdb, nlj, smj, sort_tpch, tpcds, tpch,
};
#[derive(Debug, Parser)]
@@ -46,6 +46,7 @@ struct Cli {
enum Options {
Cancellation(cancellation::RunOpt),
Clickbench(clickbench::RunOpt),
+ Dict(dict::RunOpt),
H2o(h2o::RunOpt),
HJ(hj::RunOpt),
Imdb(imdb::RunOpt),
@@ -66,6 +67,7 @@ pub async fn main() -> Result<()> {
match cli.command {
Options::Cancellation(opt) => opt.run().await,
Options::Clickbench(opt) => opt.run().await,
+ Options::Dict(opt) => opt.run().await,
Options::H2o(opt) => opt.run().await,
Options::HJ(opt) => opt.run().await,
Options::Imdb(opt) => Box::pin(opt.run()).await,
diff --git a/benchmarks/src/dict.rs b/benchmarks/src/dict.rs
new file mode 100644
index 0000000000..f8451715ea
--- /dev/null
+++ b/benchmarks/src/dict.rs
@@ -0,0 +1,436 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::util::BenchmarkRun;
+use crate::util::CommonOpt;
+use crate::util::QueryResult;
+use arrow::array::{ArrayRef, DictionaryArray, Int32Array, ListArray, StringArray};
+use arrow::buffer::OffsetBuffer;
+use arrow::datatypes::{DataType, Field, Int32Type, Schema};
+use arrow::record_batch::RecordBatch;
+use datafusion::common::DataFusionError;
+use datafusion::physical_plan::execute_stream;
+use datafusion::prelude::SessionContext;
+use datafusion_common::Result;
+use datafusion_common::exec_err;
+use datafusion_common::instant::Instant;
+use futures::StreamExt;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+const ITEMS_PER_VALUE: usize = 4;
+
+#[derive(Debug, Clone, clap::Parser)]
+pub struct RunOpt {
+ /// Number of rows in the generated table.
+ #[clap(long, default_value = "1000000")]
+ pub num_rows: usize,
+
+ /// Which query to run (1-based). Omit to run all queries.
+ #[clap(long)]
+ pub query: Option<usize>,
+
+ /// Output path for the JSON benchmark summary.
+ #[arg(short = 'o', long = "output")]
+ pub output_path: Option<PathBuf>,
+
+ #[clap(flatten)]
+ pub common: CommonOpt,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DictValueType {
+ Utf8,
+ ListUtf8,
+}
+// percent of row that are null
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum NullPercent {
+ Zero,
+ Fifteen,
+}
+
+impl NullPercent {
+ pub fn null_every(self) -> Option<usize> {
+ match self {
+ NullPercent::Zero => None,
+ NullPercent::Fifteen => Some(7), // 1-in-7 ≈ 14.3 %
+ }
+ }
+}
+
+/// Fraction of `size` that becomes the number of distinct dictionary entries.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Cardinality {
+ Five,
+ Ten,
+ TwentyFive,
+}
+
+impl Cardinality {
+ pub fn num_distinct(self, size: usize) -> usize {
+ match self {
+ Cardinality::Five => (size / 20).max(1),
+ Cardinality::Ten => (size / 10).max(1),
+ Cardinality::TwentyFive => (size / 4).max(1),
+ }
+ }
+}
+
+/// `generate_dict` arguments for one dictionary column.
+#[derive(Debug)]
+pub struct DictColParams {
+ pub value_type: DictValueType,
+ pub null_percent: NullPercent,
+ pub cardinality: Cardinality,
+}
+
+#[derive(Debug)]
+pub struct DictionaryQuery {
+ pub name: &'static str,
+ pub col: DictColParams,
+ /// Params for a second dictionary column; `None` for single-column queries.
+ pub col2: Option<DictColParams>,
+ pub sql: &'static str,
+}
+
+pub const DICTIONARY_QUERIES: &[DictionaryQuery] = &[
+ // single-column group-by: Utf8
+ DictionaryQuery {
+ name: "group_by_utf8_card5_no_nulls",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::Five,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ DictionaryQuery {
+ name: "group_by_utf8_card10_no_nulls",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::Ten,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ DictionaryQuery {
+ name: "group_by_utf8_card25_no_nulls",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::TwentyFive,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ DictionaryQuery {
+ name: "group_by_utf8_card5_null15",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Fifteen,
+ cardinality: Cardinality::Five,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ DictionaryQuery {
+ name: "group_by_utf8_card25_null15",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Fifteen,
+ cardinality: Cardinality::TwentyFive,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ // currently not supported by GroupValuesRows,
+ // https://github.com/apache/datafusion/pull/21765 Intends to address this.
+ // commenting out these benchmarks fornow
+ /*
+ DictionaryQuery {
+ name: "group_by_list_utf8_card5_no_nulls",
+ col: DictColParams {
+ value_type: DictValueType::ListUtf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::Five,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ DictionaryQuery {
+ name: "group_by_list_utf8_card10_no_nulls",
+ col: DictColParams {
+ value_type: DictValueType::ListUtf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::Ten,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ DictionaryQuery {
+ name: "group_by_list_utf8_card25_null15",
+ col: DictColParams {
+ value_type: DictValueType::ListUtf8,
+ null_percent: NullPercent::Fifteen,
+ cardinality: Cardinality::TwentyFive,
+ },
+ col2: None,
+ sql: r#"SELECT dict_col, COUNT(*) FROM test_data GROUP BY dict_col"#,
+ },
+ */
+ DictionaryQuery {
+ name: "group_by_two_utf8_card5_no_nulls",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::Five,
+ },
+ col2: Some(DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Fifteen,
+ cardinality: Cardinality::Ten,
+ }),
+ sql: r#"SELECT dict_col, dict_col2, COUNT(*) FROM test_data GROUP BY dict_col, dict_col2"#,
+ },
+ DictionaryQuery {
+ name: "group_by_two_utf8_card25_null15",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Fifteen,
+ cardinality: Cardinality::TwentyFive,
+ },
+ col2: Some(DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::Five,
+ }),
+ sql: r#"SELECT dict_col, dict_col2, COUNT(*) FROM test_data GROUP BY dict_col, dict_col2"#,
+ },
+ // --- multi-column group-by: Utf8 + List<Utf8> ----------------------------
+ /*
+ DictionaryQuery {
+ name: "group_by_utf8_and_list_utf8_card10_null15",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Fifteen,
+ cardinality: Cardinality::Ten,
+ },
+ col2: Some(DictColParams {
+ value_type: DictValueType::ListUtf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::Five,
+ }),
+ sql: r#"SELECT dict_col, dict_col2, COUNT(*) FROM test_data GROUP BY dict_col, dict_col2"#,
+ },
+ DictionaryQuery {
+ name: "group_by_utf8_and_list_utf8_card25_no_nulls",
+ col: DictColParams {
+ value_type: DictValueType::Utf8,
+ null_percent: NullPercent::Zero,
+ cardinality: Cardinality::TwentyFive,
+ },
+ col2: Some(DictColParams {
+ value_type: DictValueType::ListUtf8,
+ null_percent: NullPercent::Fifteen,
+ cardinality: Cardinality::Ten,
+ }),
+ sql: r#"SELECT dict_col, dict_col2, COUNT(*) FROM test_data GROUP BY dict_col, dict_col2"#,
+ },
+ */
+];
+
+pub fn generate_dict(
+ value_type: DictValueType,
+ cardinality: Cardinality,
+ null_percent: NullPercent,
+ size: usize,
+) -> ArrayRef {
+ let num_distinct = cardinality.num_distinct(size);
+
+ let dict_values: ArrayRef = match value_type {
+ DictValueType::Utf8 => {
+ let strings: StringArray = (0..num_distinct)
+ .map(|i| format!("value_{i}"))
+ .collect::<Vec<_>>()
+ .into();
+ Arc::new(strings)
+ }
+ DictValueType::ListUtf8 => {
+ let flat: StringArray = (0..num_distinct)
+ .flat_map(|i| (0..ITEMS_PER_VALUE).map(move |j| format!("value_{i}_{j}")))
+ .collect::<Vec<_>>()
+ .into();
+
+ let offsets: Vec<i32> = (0..=(num_distinct * ITEMS_PER_VALUE))
+ .step_by(ITEMS_PER_VALUE)
+ .map(|o| o as i32)
+ .collect();
+
+ Arc::new(ListArray::new(
+ Arc::new(Field::new("item", DataType::Utf8, false)),
+ OffsetBuffer::new(offsets.into()),
+ Arc::new(flat),
+ None,
+ ))
+ }
+ };
+
+ let null_every = null_percent.null_every();
+
+ let mut key_builder = Int32Array::builder(size);
+ for i in 0..size {
+ if null_every.is_some_and(|n| i % n == 0) {
+ key_builder.append_null();
+ } else {
+ key_builder.append_value((i % num_distinct) as i32);
+ }
+ }
+ let keys = key_builder.finish();
+
+ Arc::new(
+ DictionaryArray::<Int32Type>::try_new(keys, dict_values)
+ .expect("valid dictionary array"),
+ )
+}
+impl RunOpt {
+ pub async fn run(self) -> Result<()> {
+ println!(
+ "Running dictionary encoding benchmarks with the following options: {self:#?}\n"
+ );
+
+ let query_range = match self.query {
+ Some(query_id) => {
+ if query_id >= 1 && query_id <= DICTIONARY_QUERIES.len() {
+ query_id..=query_id
+ } else {
+ return exec_err!(
+ "Query {query_id} not found. Available queries: 1 to {}",
+ DICTIONARY_QUERIES.len()
+ );
+ }
+ }
+ None => 1..=DICTIONARY_QUERIES.len(),
+ };
+
+ let config = self.common.config()?;
+ let rt = self.common.build_runtime()?;
+ let ctx = SessionContext::new_with_config_rt(config, rt);
+ let mut benchmark_run = BenchmarkRun::new();
+
+ for query_id in query_range {
+ let query = &DICTIONARY_QUERIES[query_id - 1];
+ benchmark_run.start_new_case(query.name);
+
+ let query_run = self.benchmark_query(query, &ctx).await;
+ match query_run {
+ Ok(query_results) => {
+ for iter in query_results {
+ benchmark_run.write_iter(iter.elapsed, iter.row_count);
+ }
+ }
+ Err(e) => {
+ return Err(DataFusionError::Context(
+ format!("Dictionary benchmark '{}' failed:", query.name),
+ Box::new(e),
+ ));
+ }
+ }
+ }
+
+ benchmark_run.maybe_write_json(self.output_path.as_ref())?;
+ Ok(())
+ }
+
+ async fn benchmark_query(
+ &self,
+ query: &DictionaryQuery,
+ ctx: &SessionContext,
+ ) -> Result<Vec<QueryResult>> {
+ let batch = self.make_record_batch(query)?;
+ ctx.deregister_table("test_data")?;
+ ctx.register_batch("test_data", batch)?;
+
+ let mut query_results = vec![];
+ for i in 0..self.common.iterations {
+ let start = Instant::now();
+ let row_count =
+ Self::execute_query_without_result_buffering(query.sql, ctx).await?;
+ let elapsed = start.elapsed();
+ println!(
+ "Query '{}' iteration {i} returned {row_count} rows in {elapsed:?}",
+ query.name
+ );
+ query_results.push(QueryResult { elapsed, row_count });
+ }
+
+ Ok(query_results)
+ }
+
+ fn make_record_batch(&self, query: &DictionaryQuery) -> Result<RecordBatch> {
+ let size = self.num_rows;
+
+ let col1 = generate_dict(
+ query.col.value_type,
+ query.col.cardinality,
+ query.col.null_percent,
+ size,
+ );
+
+ let (schema, columns): (Schema, Vec<ArrayRef>) = match &query.col2 {
+ None => {
+ let schema = Schema::new(vec![Field::new(
+ "dict_col",
+ col1.data_type().clone(),
+ true,
+ )]);
+ (schema, vec![col1])
+ }
+ Some(col2_params) => {
+ let col2 = generate_dict(
+ col2_params.value_type,
+ col2_params.cardinality,
+ col2_params.null_percent,
+ size,
+ );
+ let schema = Schema::new(vec![
+ Field::new("dict_col", col1.data_type().clone(), true),
+ Field::new("dict_col2", col2.data_type().clone(), true),
+ ]);
+ (schema, vec![col1, col2])
+ }
+ };
+
+ Ok(RecordBatch::try_new(Arc::new(schema), columns)?)
+ }
+
+ async fn execute_query_without_result_buffering(
+ sql: &str,
+ ctx: &SessionContext,
+ ) -> Result<usize> {
+ let mut row_count = 0;
+ let df = ctx.sql(sql).await?;
+ let physical_plan = df.create_physical_plan().await?;
+ let mut stream = execute_stream(physical_plan, ctx.task_ctx())?;
+ while let Some(batch) = stream.next().await {
+ row_count += batch?.num_rows();
+ }
+ Ok(row_count)
+ }
+}
diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs
index 983c5fc24a..f41fd5ebed 100644
--- a/benchmarks/src/lib.rs
+++ b/benchmarks/src/lib.rs
@@ -19,6 +19,7 @@
pub mod benchmark_runner;
pub mod cancellation;
pub mod clickbench;
+pub mod dict;
pub mod h2o;
pub mod hj;
pub mod imdb;