Skip to content

Apache DataFusion SQL Query Engine

perf: improve Int64 `generate_series` and `range` performance (#21891)

linfeng0305098d583d
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml
index b25aa0b9bc..d20542daab 100644
--- a/datafusion/functions-nested/Cargo.toml
+++ b/datafusion/functions-nested/Cargo.toml
@@ -133,3 +133,7 @@ name = "string_to_array"
[[bench]]
harness = false
name = "array_resize"
+
+[[bench]]
+harness = false
+name = "array_range"
diff --git a/datafusion/functions-nested/benches/array_range.rs b/datafusion/functions-nested/benches/array_range.rs
new file mode 100644
index 0000000000..1f82cbd229
--- /dev/null
+++ b/datafusion/functions-nested/benches/array_range.rs
@@ -0,0 +1,208 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmarks for range and generate_series functions.
+
+use arrow::array::{ArrayRef, Int64Array};
+use arrow::datatypes::{DataType, Field};
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use datafusion_common::{ScalarValue, config::ConfigOptions};
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+use datafusion_functions_nested::range::{Range, gen_series_udf};
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use std::hint::black_box;
+use std::sync::Arc;
+
+const NUM_ROWS: &[usize] = &[100, 1000, 10000];
+const INT_STEPS: &[i64] = &[1, 5, 50];
+const INT_DIRECTIONS: &[(&str, bool)] = &[("increasing", true), ("decreasing", false)];
+/// Each row produces at most RANGE_SIZE elements in its Int64 list
+const RANGE_SIZE: i64 = 200;
+const SEED: u64 = 42;
+
+fn criterion_benchmark(c: &mut Criterion) {
+ bench_range_int64(c);
+ bench_generate_series_int64(c);
+}
+
+// ---------------------------------------------------------------------------
+// Int64 – range(start, stop, step)
+// ---------------------------------------------------------------------------
+fn bench_range_int64(c: &mut Criterion) {
+ let mut group = c.benchmark_group("range_int64");
+
+ for &num_rows in NUM_ROWS {
+ for &(direction, increasing) in INT_DIRECTIONS {
+ let (start_array, stop_array) =
+ make_int64_start_stop_arrays(num_rows, RANGE_SIZE, increasing);
+
+ for &step in INT_STEPS {
+ let step = if increasing { step } else { -step };
+ let args = vec![
+ ColumnarValue::Array(start_array.clone()),
+ ColumnarValue::Array(stop_array.clone()),
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(step))),
+ ];
+
+ group.bench_with_input(
+ BenchmarkId::new(format!("{direction}/step_{step}"), num_rows),
+ &num_rows,
+ |b, _| {
+ let udf = Range::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: vec![
+ Arc::new(Field::new(
+ "start",
+ DataType::Int64,
+ true,
+ )),
+ Arc::new(Field::new(
+ "stop",
+ DataType::Int64,
+ true,
+ )),
+ Arc::new(Field::new(
+ "step",
+ DataType::Int64,
+ true,
+ )),
+ ],
+ number_rows: num_rows,
+ return_field: Arc::new(Field::new(
+ "result",
+ DataType::List(Arc::new(Field::new_list_field(
+ DataType::Int64,
+ true,
+ ))),
+ true,
+ )),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ },
+ );
+ }
+ }
+ }
+
+ group.finish();
+}
+
+// ---------------------------------------------------------------------------
+// Int64 – generate_series(start, stop, step)
+// ---------------------------------------------------------------------------
+fn bench_generate_series_int64(c: &mut Criterion) {
+ let mut group = c.benchmark_group("generate_series_int64");
+
+ for &num_rows in NUM_ROWS {
+ for &(direction, increasing) in INT_DIRECTIONS {
+ let (start_array, stop_array) =
+ make_int64_start_stop_arrays(num_rows, RANGE_SIZE, increasing);
+
+ for &step in INT_STEPS {
+ let step = if increasing { step } else { -step };
+ let args = vec![
+ ColumnarValue::Array(start_array.clone()),
+ ColumnarValue::Array(stop_array.clone()),
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(step))),
+ ];
+
+ group.bench_with_input(
+ BenchmarkId::new(format!("{direction}/step_{step}"), num_rows),
+ &num_rows,
+ |b, _| {
+ let udf = gen_series_udf();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: vec![
+ Arc::new(Field::new(
+ "start",
+ DataType::Int64,
+ true,
+ )),
+ Arc::new(Field::new(
+ "stop",
+ DataType::Int64,
+ true,
+ )),
+ Arc::new(Field::new(
+ "step",
+ DataType::Int64,
+ true,
+ )),
+ ],
+ number_rows: num_rows,
+ return_field: Arc::new(Field::new(
+ "result",
+ DataType::List(Arc::new(Field::new_list_field(
+ DataType::Int64,
+ true,
+ ))),
+ true,
+ )),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ },
+ );
+ }
+ }
+ }
+
+ group.finish();
+}
+
+/// Build (start, stop) Int64Arrays where each stop = start + offset,
+/// with offset in [1, max_range]. This ensures every row produces a
+/// bounded-size list, avoiding OOM from unbounded i64 ranges.
+fn make_int64_start_stop_arrays(
+ num_rows: usize,
+ max_range: i64,
+ increasing: bool,
+) -> (ArrayRef, ArrayRef) {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let mut starts: Vec<i64> = Vec::with_capacity(num_rows);
+ let mut stops: Vec<i64> = Vec::with_capacity(num_rows);
+ for _ in 0..num_rows {
+ let s = rng.random_range(0..max_range);
+ let offset = rng.random_range(1..=max_range);
+ if increasing {
+ starts.push(s);
+ stops.push(s + offset);
+ } else {
+ starts.push(s + offset);
+ stops.push(s);
+ }
+ }
+ (
+ Arc::new(Int64Array::from(starts)),
+ Arc::new(Int64Array::from(stops)),
+ )
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/functions-nested/src/range.rs b/datafusion/functions-nested/src/range.rs
index c51b008191..72450cb56b 100644
--- a/datafusion/functions-nested/src/range.rs
+++ b/datafusion/functions-nested/src/range.rs
@@ -33,8 +33,7 @@ use arrow::{
};
use datafusion_common::internal_err;
use datafusion_common::{
- Result, exec_datafusion_err, exec_err, not_impl_datafusion_err,
- utils::take_function_args,
+ Result, exec_datafusion_err, exec_err, utils::take_function_args,
};
use datafusion_common::{
ScalarValue,
@@ -324,15 +323,12 @@ impl Range {
);
}
Some((start, stop, step)) => {
- // Below, we utilize `usize` to represent steps.
- // On 32-bit targets, the absolute value of `i64` may fail to fit into `usize`.
- let step_abs =
- usize::try_from(step.unsigned_abs()).map_err(|_| {
- not_impl_datafusion_err!("step {} can't fit into usize", step)
- })?;
- values.extend(
- gen_range_iter(start, stop, step < 0, self.include_upper_bound)
- .step_by(step_abs),
+ generate_range_values(
+ start,
+ stop,
+ step,
+ self.include_upper_bound,
+ &mut values,
);
offsets.push(values.len() as i32);
valid.append_non_null();
@@ -542,32 +538,59 @@ fn retrieve_range_args(
Some((start, stop, step))
}
-/// Returns an iterator of i64 values from start to stop
-fn gen_range_iter(
+/// Generate integer range values directly into the provided buffer.
+#[inline]
+fn generate_range_values(
start: i64,
stop: i64,
- decreasing: bool,
+ step: i64,
include_upper: bool,
-) -> Box<dyn Iterator<Item = i64>> {
- match (decreasing, include_upper) {
- // Decreasing range, stop is inclusive
- (true, true) => Box::new((stop..=start).rev()),
- // Decreasing range, stop is exclusive
- (true, false) => {
- if stop == i64::MAX {
- // start is never greater than stop, and stop is exclusive,
- // so the decreasing range must be empty.
- Box::new(std::iter::empty())
- } else {
- // Increase the stop value by one to exclude it.
- // Since stop is not i64::MAX, `stop + 1` will not overflow.
- Box::new((stop + 1..=start).rev())
+ values: &mut Vec<i64>,
+) {
+ if !include_upper && start == stop {
+ return;
+ }
+
+ if step > 0 {
+ let limit = if include_upper {
+ stop
+ } else {
+ stop.saturating_sub(1)
+ };
+ if start > limit {
+ return;
+ }
+ let count =
+ (start.abs_diff(limit) / step.unsigned_abs()).saturating_add(1) as usize;
+ values.reserve(count);
+ let mut current = start;
+ while current <= limit {
+ values.push(current);
+ match current.checked_add(step) {
+ Some(next) => current = next,
+ None => break,
+ }
+ }
+ } else if step < 0 {
+ let limit = if include_upper {
+ stop
+ } else {
+ stop.saturating_add(1)
+ };
+ if start < limit {
+ return;
+ }
+ let count =
+ (start.abs_diff(limit) / step.unsigned_abs()).saturating_add(1) as usize;
+ values.reserve(count);
+ let mut current = start;
+ while current >= limit {
+ values.push(current);
+ match current.checked_add(step) {
+ Some(next) => current = next,
+ None => break,
}
}
- // Increasing range, stop is inclusive
- (false, true) => Box::new(start..=stop),
- // Increasing range, stop is exclusive
- (false, false) => Box::new(start..stop),
}
}
diff --git a/datafusion/sqllogictest/test_files/array/array_range.slt b/datafusion/sqllogictest/test_files/array/array_range.slt
index cb73485581..a55ec4a657 100644
--- a/datafusion/sqllogictest/test_files/array/array_range.slt
+++ b/datafusion/sqllogictest/test_files/array/array_range.slt
@@ -48,6 +48,16 @@ select
----
[] [] [0] [0]
+# Test range with full-span bounds and large steps
+query ????
+select
+ range(-9223372036854775808, 9223372036854775807, 9223372036854775807) as c1,
+ range(9223372036854775807, -9223372036854775808, -9223372036854775807) as c2,
+ generate_series(-9223372036854775808, 9223372036854775807, 9223372036854775807) as c3,
+ generate_series(9223372036854775807, -9223372036854775808, -9223372036854775807) as c4;
+----
+[-9223372036854775808, -1, 9223372036854775806] [9223372036854775807, 0, -9223372036854775807] [-9223372036854775808, -1, 9223372036854775806] [9223372036854775807, 0, -9223372036854775807]
+
# Test range for other edge cases
query ????????
select