Apache DataFusion SQL Query Engine
perf: optimize retract_batch for `median` and `percentile_cont` (#21894)
lynee514a011192e
diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.tomlindex 406f0a0e32..778e6a24bf 100644--- a/datafusion/functions-aggregate/Cargo.toml+++ b/datafusion/functions-aggregate/Cargo.toml@@ -87,3 +87,11 @@ harness = false[[bench]]name = "count_distinct"harness = false++[[bench]]+name = "median"+harness = false++[[bench]]+name = "percentile_cont"+harness = falsediff --git a/datafusion/functions-aggregate/benches/median.rs b/datafusion/functions-aggregate/benches/median.rsnew file mode 100644index 0000000000..0f5f70c7b4--- /dev/null+++ b/datafusion/functions-aggregate/benches/median.rs@@ -0,0 +1,122 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::hint::black_box;+use std::sync::Arc;++use arrow::array::{ArrayRef, Float64Array};+use arrow::datatypes::{DataType, Field, Schema};+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};+use datafusion_expr::function::AccumulatorArgs;+use datafusion_expr::{Accumulator, AggregateUDFImpl};+use datafusion_functions_aggregate::median::Median;+use datafusion_physical_expr::expressions::col;++const STEP_SIZE: usize = 128;+const SLIDES_PER_ITER: usize = 32;+const WINDOW_SIZES: [usize; 3] = [256, 4096, 16384];++fn prepare_accumulator() -> Box<dyn Accumulator> {+ let schema = Arc::new(Schema::new(vec![Field::new("f", DataType::Float64, true)]));+ let expr = col("f", &schema).unwrap();+ let accumulator_args = AccumulatorArgs {+ return_field: Field::new("f", DataType::Float64, true).into(),+ schema: &schema,+ expr_fields: &[expr.return_field(&schema).unwrap()],+ ignore_nulls: false,+ order_bys: &[],+ is_reversed: false,+ name: "median(f)",+ is_distinct: false,+ exprs: &[expr],+ };+ Median::new().accumulator(accumulator_args).unwrap()+}++fn stream_array(len: usize, null_stride: Option<usize>) -> ArrayRef {+ let values = (0..len)+ .map(|idx| {+ if null_stride.is_some_and(|stride| idx % stride == 0) {+ None+ } else {+ Some(idx as f64)+ }+ })+ .collect::<Vec<_>>();+ Arc::new(Float64Array::from(values)) as ArrayRef+}++/// Benchmark the sliding window cycle: retract + update + evaluate+fn sliding_window_bench(+ c: &mut Criterion,+ name: &str,+ window_size: usize,+ stream: &ArrayRef,+) {+ c.bench_function(name, |b| {+ b.iter_batched(+ || {+ let mut accumulator = prepare_accumulator();+ let initial = stream.slice(0, window_size);+ accumulator+ .update_batch(std::slice::from_ref(&initial))+ .unwrap();+ accumulator+ },+ |mut accumulator| {+ for slide in 0..SLIDES_PER_ITER {+ let offset = slide * STEP_SIZE;+ let retract = stream.slice(offset, STEP_SIZE);+ let update = stream.slice(offset + window_size, STEP_SIZE);+ accumulator+ .retract_batch(std::slice::from_ref(&retract))+ .unwrap();+ accumulator+ .update_batch(std::slice::from_ref(&update))+ .unwrap();+ black_box(accumulator.evaluate().unwrap());+ }+ },+ BatchSize::SmallInput,+ )+ });+}++fn median_benchmark(c: &mut Criterion) {+ for window_size in WINDOW_SIZES {+ let stream_len = window_size + STEP_SIZE * SLIDES_PER_ITER;+ let stream_no_nulls = stream_array(stream_len, None);+ let stream_with_nulls = stream_array(stream_len, Some(10));++ sliding_window_bench(+ c,+ &format!("median sliding_window f64 no_nulls window_size={window_size}"),+ window_size,+ &stream_no_nulls,+ );++ sliding_window_bench(+ c,+ &format!("median sliding_window f64 with_nulls window_size={window_size}"),+ window_size,+ &stream_with_nulls,+ );+ }+}++criterion_group!(benches, median_benchmark);+criterion_main!(benches);diff --git a/datafusion/functions-aggregate/benches/percentile_cont.rs b/datafusion/functions-aggregate/benches/percentile_cont.rsnew file mode 100644index 0000000000..05119441e1--- /dev/null+++ b/datafusion/functions-aggregate/benches/percentile_cont.rs@@ -0,0 +1,129 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::hint::black_box;+use std::sync::Arc;++use arrow::array::{ArrayRef, Float64Array};+use arrow::datatypes::{DataType, Field, Schema};+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};+use datafusion_expr::function::AccumulatorArgs;+use datafusion_expr::{Accumulator, AggregateUDFImpl};+use datafusion_functions_aggregate::percentile_cont::PercentileCont;+use datafusion_physical_expr::expressions::{col, lit};++const STEP_SIZE: usize = 128;+const SLIDES_PER_ITER: usize = 32;+const WINDOW_SIZES: [usize; 3] = [256, 4096, 16384];++fn prepare_accumulator() -> Box<dyn Accumulator> {+ let schema = Arc::new(Schema::new(vec![Field::new("f", DataType::Float64, true)]));+ let value_expr = col("f", &schema).unwrap();+ let percentile_expr = lit(0.5_f64);+ let value_field = value_expr.return_field(&schema).unwrap();+ let percentile_field = percentile_expr.return_field(&schema).unwrap();+ let accumulator_args = AccumulatorArgs {+ return_field: Field::new("f", DataType::Float64, true).into(),+ schema: &schema,+ expr_fields: &[value_field, percentile_field],+ ignore_nulls: false,+ order_bys: &[],+ is_reversed: false,+ name: "percentile_cont(f, 0.5)",+ is_distinct: false,+ exprs: &[value_expr, percentile_expr],+ };+ PercentileCont::new().accumulator(accumulator_args).unwrap()+}++fn stream_array(len: usize, null_stride: Option<usize>) -> ArrayRef {+ let values = (0..len)+ .map(|idx| {+ if null_stride.is_some_and(|stride| idx % stride == 0) {+ None+ } else {+ Some(idx as f64)+ }+ })+ .collect::<Vec<_>>();+ Arc::new(Float64Array::from(values)) as ArrayRef+}++/// Benchmark the sliding window cycle: retract + update + evaluate+fn sliding_window_bench(+ c: &mut Criterion,+ name: &str,+ window_size: usize,+ stream: &ArrayRef,+) {+ c.bench_function(name, |b| {+ b.iter_batched(+ || {+ let mut accumulator = prepare_accumulator();+ let initial = stream.slice(0, window_size);+ accumulator+ .update_batch(std::slice::from_ref(&initial))+ .unwrap();+ accumulator+ },+ |mut accumulator| {+ for slide in 0..SLIDES_PER_ITER {+ let offset = slide * STEP_SIZE;+ let retract = stream.slice(offset, STEP_SIZE);+ let update = stream.slice(offset + window_size, STEP_SIZE);+ accumulator+ .retract_batch(std::slice::from_ref(&retract))+ .unwrap();+ accumulator+ .update_batch(std::slice::from_ref(&update))+ .unwrap();+ black_box(accumulator.evaluate().unwrap());+ }+ },+ BatchSize::SmallInput,+ )+ });+}++fn percentile_cont_benchmark(c: &mut Criterion) {+ for window_size in WINDOW_SIZES {+ let stream_len = window_size + STEP_SIZE * SLIDES_PER_ITER;+ let stream_no_nulls = stream_array(stream_len, None);+ let stream_with_nulls = stream_array(stream_len, Some(10));++ sliding_window_bench(+ c,+ &format!(+ "percentile_cont sliding_window f64 no_nulls window_size={window_size}"+ ),+ window_size,+ &stream_no_nulls,+ );++ sliding_window_bench(+ c,+ &format!(+ "percentile_cont sliding_window f64 with_nulls window_size={window_size}"+ ),+ window_size,+ &stream_with_nulls,+ );+ }+}++criterion_group!(benches, percentile_cont_benchmark);+criterion_main!(benches);diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rsindex ab6d9e1832..74473e2f28 100644--- a/datafusion/functions-aggregate/src/median.rs+++ b/datafusion/functions-aggregate/src/median.rs@@ -51,7 +51,7 @@ use datafusion_expr::{use datafusion_expr::{EmitTo, GroupsAccumulator};use datafusion_functions_aggregate_common::aggregate::groups_accumulator::accumulate::accumulate;use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::filtered_null_mask;-use datafusion_functions_aggregate_common::utils::GenericDistinctBuffer;+use datafusion_functions_aggregate_common::utils::{GenericDistinctBuffer, Hashable};use datafusion_macros::user_doc;use std::collections::HashMap;@@ -285,24 +285,17 @@ impl<T: ArrowNumericType> Accumulator for MedianAccumulator<T> {size_of_val(self) + self.all_values.capacity() * size_of::<T::Native>()}- #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash keyfn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- let mut to_remove: HashMap<ScalarValue, usize> = HashMap::new();+ let mut to_remove: HashMap<Hashable<T::Native>, usize> = HashMap::new();- let arr = &values[0];- for i in 0..arr.len() {- let v = ScalarValue::try_from_array(arr, i)?;- if !v.is_null() {- *to_remove.entry(v).or_default() += 1;- }+ let arr = values[0].as_primitive::<T>();+ for value in arr.iter().flatten() {+ *to_remove.entry(Hashable(value)).or_default() += 1;}let mut i = 0;while i < self.all_values.len() {- let k = ScalarValue::new_primitive::<T>(- Some(self.all_values[i]),- &self.data_type,- )?;+ let k = Hashable(self.all_values[i]);if let Some(count) = to_remove.get_mut(&k)&& *count > 0{diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rsindex bbf6821ba0..256388c216 100644--- a/datafusion/functions-aggregate/src/percentile_cont.rs+++ b/datafusion/functions-aggregate/src/percentile_cont.rs@@ -440,20 +440,17 @@ wheresize_of_val(self) + self.all_values.capacity() * size_of::<T::Native>()}- #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash keyfn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- let mut to_remove: HashMap<ScalarValue, usize> = HashMap::new();- for i in 0..values[0].len() {- let v = ScalarValue::try_from_array(&values[0], i)?;- if !v.is_null() {- *to_remove.entry(v).or_default() += 1;- }+ let mut to_remove: HashMap<Hashable<T::Native>, usize> = HashMap::new();++ let arr = values[0].as_primitive::<T>();+ for value in arr.iter().flatten() {+ *to_remove.entry(Hashable(value)).or_default() += 1;}let mut i = 0;while i < self.all_values.len() {- let k =- ScalarValue::new_primitive::<T>(Some(self.all_values[i]), &T::DATA_TYPE)?;+ let k = Hashable(self.all_values[i]);if let Some(count) = to_remove.get_mut(&k)&& *count > 0{