Apache DataFusion SQL Query Engine
Add sorted data benchmark. (#19042)
Qi Zhucde6dfab1c5c
diff --git a/benchmarks/README.md b/benchmarks/README.mdindex bf9b50028b..c84af9a7e6 100644--- a/benchmarks/README.md+++ b/benchmarks/README.md@@ -832,3 +832,41 @@ Getting results...cancelling threaddone dropping runtime in 83.531417ms```++## Sorted Data Benchmarks++### Data Sorted ClickBench++Benchmark for queries on pre-sorted data to test sort order optimization.+This benchmark uses a subset of the ClickBench dataset (hits.parquet, ~14GB) that has been pre-sorted by the EventTime column. The queries are designed to test DataFusion's performance when the data is already sorted as is common in timeseries workloads.++The benchmark includes queries that:+- Scan pre-sorted data with ORDER BY clauses that match the sort order+- Test reverse scans on sorted data+- Verify the performance result++#### Generating Sorted Data++The sorted dataset is automatically generated from the ClickBench partitioned dataset. You can configure the memory used during the sorting process with the `DATAFUSION_MEMORY_GB` environment variable. The default memory limit is 12GB.+```bash+./bench.sh data data_sorted_clickbench+```++To create the sorted dataset, for example with 16GB of memory, run:++```bash+DATAFUSION_MEMORY_GB=16 ./bench.sh data data_sorted_clickbench+```++This command will:+1. Download the ClickBench partitioned dataset if not present+2. Sort hits.parquet by EventTime in ascending order+3. Save the sorted file as hits_sorted.parquet++#### Running the Benchmark++```bash+./bench.sh run data_sorted_clickbench+```++This runs queries against the pre-sorted dataset with the `--sorted-by EventTime` flag, which informs DataFusion that the data is pre-sorted, allowing it to optimize away redundant sort operations.diff --git a/benchmarks/bench.sh b/benchmarks/bench.shindex 6b94dadeb9..a91a6c32bd 100755--- a/benchmarks/bench.sh+++ b/benchmarks/bench.sh@@ -102,6 +102,9 @@ clickbench_partitioned: ClickBench queries against partitioned (100 files) parquclickbench_pushdown: ClickBench queries against partitioned (100 files) parquet w/ filter_pushdown enabledclickbench_extended: ClickBench \"inspired\" queries against a single parquet (DataFusion specific)+# Sorted Data Benchmarks (ORDER BY Optimization)+clickbench_sorted: ClickBench queries on pre-sorted data using prefer_existing_sort (tests sort elimination optimization)+# H2O.ai Benchmarks (Group By, Join, Window)h2o_small: h2oai benchmark with small dataset (1e7 rows) for groupby, default file format is csvh2o_medium: h2oai benchmark with medium dataset (1e8 rows) for groupby, default file format is csv@@ -324,6 +327,9 @@ main() {compile_profile)data_tpch "1" "parquet";;+ clickbench_sorted)+ clickbench_sorted+ ;;*)echo "Error: unknown benchmark '$BENCHMARK' for data generation"usage@@ -459,7 +465,7 @@ main() {h2o_medium_window)run_h2o_window "MEDIUM" "CSV" "window";;- h2o_big_window)+ h2o_big_window)run_h2o_window "BIG" "CSV" "window";;h2o_small_parquet)@@ -511,6 +517,9 @@ main() {compile_profile)run_compile_profile "${PROFILE_ARGS[@]}";;+ clickbench_sorted)+ run_clickbench_sorted+ ;;*)echo "Error: unknown benchmark '$BENCHMARK' for run"usage@@ -1260,6 +1269,113 @@ compare_benchmarks() {}+# Creates sorted ClickBench data from hits.parquet (full dataset)+# The data is sorted by EventTime in ascending order+# Uses datafusion-cli to reduce dependencies+clickbench_sorted() {+ SORTED_FILE="${DATA_DIR}/hits_sorted.parquet"+ ORIGINAL_FILE="${DATA_DIR}/hits.parquet"++ # Default memory limit is 12GB, can be overridden with DATAFUSION_MEMORY_GB env var+ MEMORY_LIMIT_GB=${DATAFUSION_MEMORY_GB:-12}++ echo "Creating sorted ClickBench dataset from hits.parquet..."+ echo "Configuration:"+ echo " Memory limit: ${MEMORY_LIMIT_GB}G"+ echo " Row group size: 64K rows"+ echo " Compression: uncompressed"++ if [ ! -f "${ORIGINAL_FILE}" ]; then+ echo "hits.parquet not found. Running data_clickbench_1 first..."+ data_clickbench_1+ fi++ if [ -f "${SORTED_FILE}" ]; then+ echo "Sorted hits.parquet already exists at ${SORTED_FILE}"+ return 0+ fi++ echo "Sorting hits.parquet by EventTime (this may take several minutes)..."++ pushd "${DATAFUSION_DIR}" > /dev/null+ echo "Building datafusion-cli..."+ cargo build --release --bin datafusion-cli+ DATAFUSION_CLI="${DATAFUSION_DIR}/target/release/datafusion-cli"+ popd > /dev/null+++ START_TIME=$(date +%s)+ echo "Start time: $(date '+%Y-%m-%d %H:%M:%S')"+ echo "Using datafusion-cli to create sorted parquet file..."+ "${DATAFUSION_CLI}" << EOF+-- Memory and performance configuration+SET datafusion.runtime.memory_limit = '${MEMORY_LIMIT_GB}G';+SET datafusion.execution.spill_compression = 'uncompressed';+SET datafusion.execution.sort_spill_reservation_bytes = 10485760; -- 10MB+SET datafusion.execution.batch_size = 8192;+SET datafusion.execution.target_partitions = 1;++-- Parquet output configuration+SET datafusion.execution.parquet.max_row_group_size = 65536;+SET datafusion.execution.parquet.compression = 'uncompressed';++-- Execute sort and write+COPY (SELECT * FROM '${ORIGINAL_FILE}' ORDER BY "EventTime")+TO '${SORTED_FILE}'+STORED AS PARQUET;+EOF++ local result=$?++ END_TIME=$(date +%s)+ DURATION=$((END_TIME - START_TIME))+ echo "End time: $(date '+%Y-%m-%d %H:%M:%S')"++ if [ $result -eq 0 ]; then+ echo "✓ Successfully created sorted ClickBench dataset"++ INPUT_SIZE=$(stat -f%z "${ORIGINAL_FILE}" 2>/dev/null || stat -c%s "${ORIGINAL_FILE}" 2>/dev/null)+ OUTPUT_SIZE=$(stat -f%z "${SORTED_FILE}" 2>/dev/null || stat -c%s "${SORTED_FILE}" 2>/dev/null)+ INPUT_MB=$((INPUT_SIZE / 1024 / 1024))+ OUTPUT_MB=$((OUTPUT_SIZE / 1024 / 1024))++ echo " Input: ${INPUT_MB} MB"+ echo " Output: ${OUTPUT_MB} MB"++ echo ""+ echo "Time Statistics:"+ echo " Total duration: ${DURATION} seconds ($(printf '%02d:%02d:%02d' $((DURATION/3600)) $((DURATION%3600/60)) $((DURATION%60))))"+ echo " Throughput: $((INPUT_MB / DURATION)) MB/s"++ return 0+ else+ echo "✗ Error: Failed to create sorted dataset"+ echo "💡 Tip: Try increasing memory with: DATAFUSION_MEMORY_GB=16 ./bench.sh data clickbench_sorted"+ return 1+ fi+}++# Runs the sorted data benchmark with prefer_existing_sort configuration+run_clickbench_sorted() {+ RESULTS_FILE="${RESULTS_DIR}/clickbench_sorted.json"+ echo "RESULTS_FILE: ${RESULTS_FILE}"+ echo "Running sorted data benchmark with prefer_existing_sort optimization..."++ # Ensure sorted data exists+ clickbench_sorted++ # Run benchmark with prefer_existing_sort configuration+ # This allows DataFusion to optimize away redundant sorts while maintaining parallelism+ debug_run $CARGO_COMMAND --bin dfbench -- clickbench \+ --iterations 5 \+ --path "${DATA_DIR}/hits_sorted.parquet" \+ --queries-path "${SCRIPT_DIR}/queries/clickbench/queries/sorted_data" \+ --sorted-by "EventTime" \+ -c datafusion.optimizer.prefer_existing_sort=true \+ -o "${RESULTS_FILE}" \+ ${QUERY_ARG}+}+setup_venv() {python3 -m venv "$VIRTUAL_ENV"PATH=$VIRTUAL_ENV/bin:$PATH python3 -m pip install -r requirements.txtdiff --git a/benchmarks/queries/clickbench/queries/sorted_data/q0.sql b/benchmarks/queries/clickbench/queries/sorted_data/q0.sqlnew file mode 100644index 0000000000..1170a383bc--- /dev/null+++ b/benchmarks/queries/clickbench/queries/sorted_data/q0.sql@@ -0,0 +1,3 @@+-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591+-- set datafusion.execution.parquet.binary_as_string = true+SELECT * FROM hits ORDER BY "EventTime" DESC limit 10;diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rsindex a550503390..796b6a4dec 100644--- a/benchmarks/src/clickbench.rs+++ b/benchmarks/src/clickbench.rs@@ -78,6 +78,27 @@ pub struct RunOpt {/// If present, write results json here#[structopt(parse(from_os_str), short = "o", long = "output")]output_path: Option<PathBuf>,++ /// Column name that the data is sorted by (e.g., "EventTime")+ /// If specified, DataFusion will be informed that the data has this sort order+ /// using CREATE EXTERNAL TABLE with WITH ORDER clause.+ ///+ /// Recommended to use with: -c datafusion.optimizer.prefer_existing_sort=true+ /// This allows DataFusion to optimize away redundant sorts while maintaining+ /// multi-core parallelism for other operations.+ #[structopt(long = "sorted-by")]+ sorted_by: Option<String>,++ /// Sort order: ASC or DESC (default: ASC)+ #[structopt(long = "sort-order", default_value = "ASC")]+ sort_order: String,++ /// Configuration options in the format key=value+ /// Can be specified multiple times.+ ///+ /// Example: -c datafusion.optimizer.prefer_existing_sort=true+ #[structopt(short = "c", long = "config")]+ config_options: Vec<String>,}/// Get the SQL file path@@ -125,6 +146,37 @@ impl RunOpt {// configure parquet optionslet mut config = self.common.config()?;++ if self.sorted_by.is_some() {+ println!("ℹ️ Data is registered with sort order");++ let has_prefer_sort = self+ .config_options+ .iter()+ .any(|opt| opt.contains("prefer_existing_sort=true"));++ if !has_prefer_sort {+ println!("ℹ️ Consider using -c datafusion.optimizer.prefer_existing_sort=true");+ println!("ℹ️ to optimize queries while maintaining parallelism");+ }+ }++ // Apply user-provided configuration options+ for config_opt in &self.config_options {+ let parts: Vec<&str> = config_opt.splitn(2, '=').collect();+ if parts.len() != 2 {+ return Err(exec_datafusion_err!(+ "Invalid config option format: '{}'. Expected 'key=value'",+ config_opt+ ));+ }+ let key = parts[0];+ let value = parts[1];++ println!("Setting config: {key} = {value}");+ config = config.set_str(key, value);+ }+{let parquet_options = &mut config.options_mut().execution.parquet;// The hits_partitioned dataset specifies string columns@@ -136,10 +188,18 @@ impl RunOpt {parquet_options.pushdown_filters = true;parquet_options.reorder_filters = true;}++ if self.sorted_by.is_some() {+ // We should compare the dynamic topk optimization when data is sorted, so we make the+ // assumption that filter pushdown is also enabled in this case.+ parquet_options.pushdown_filters = true;+ parquet_options.reorder_filters = true;+ }}let rt_builder = self.common.runtime_env_builder()?;let ctx = SessionContext::new_with_config_rt(config, rt_builder.build_arc()?);+self.register_hits(&ctx).await?;let mut benchmark_run = BenchmarkRun::new();@@ -214,17 +274,54 @@ impl RunOpt {}/// Registers the `hits.parquet` as a table named `hits`+ /// If sorted_by is specified, uses CREATE EXTERNAL TABLE with WITH ORDERasync fn register_hits(&self, ctx: &SessionContext) -> Result<()> {- let options = Default::default();let path = self.path.as_os_str().to_str().unwrap();- ctx.register_parquet("hits", path, options)- .await- .map_err(|e| {- DataFusionError::Context(- format!("Registering 'hits' as {path}"),- Box::new(e),- )- })++ // If sorted_by is specified, use CREATE EXTERNAL TABLE with WITH ORDER+ if let Some(ref sort_column) = self.sorted_by {+ println!(+ "Registering table with sort order: {} {}",+ sort_column, self.sort_order+ );++ // Escape column name with double quotes+ let escaped_column = if sort_column.contains('"') {+ sort_column.clone()+ } else {+ format!("\"{sort_column}\"")+ };++ // Build CREATE EXTERNAL TABLE DDL with WITH ORDER clause+ // Schema will be automatically inferred from the Parquet file+ let create_table_sql = format!(+ "CREATE EXTERNAL TABLE hits \+ STORED AS PARQUET \+ LOCATION '{}' \+ WITH ORDER ({} {})",+ path,+ escaped_column,+ self.sort_order.to_uppercase()+ );++ println!("Executing: {create_table_sql}");++ // Execute the CREATE EXTERNAL TABLE statement+ ctx.sql(&create_table_sql).await?.collect().await?;++ Ok(())+ } else {+ // Original registration without sort order+ let options = Default::default();+ ctx.register_parquet("hits", path, options)+ .await+ .map_err(|e| {+ DataFusionError::Context(+ format!("Registering 'hits' as {path}"),+ Box::new(e),+ )+ })+ }}fn iterations(&self) -> usize {