Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize `array_has()` for scalar needle (#20374)

Neil Conway0294a22cf96e
diff --git a/datafusion/functions-nested/benches/array_has.rs b/datafusion/functions-nested/benches/array_has.rs
index d96f26d410..302ef91686 100644
--- a/datafusion/functions-nested/benches/array_has.rs
+++ b/datafusion/functions-nested/benches/array_has.rs
@@ -15,19 +15,31 @@
// specific language governing permissions and limitations
// under the License.
+use arrow::array::{ArrayRef, Int64Array, ListArray, StringArray};
+use arrow::buffer::OffsetBuffer;
+use arrow::datatypes::{DataType, Field};
use criterion::{
criterion_group, criterion_main, {BenchmarkId, Criterion},
};
-use datafusion_expr::lit;
-use datafusion_functions_nested::expr_fn::{
- array_has, array_has_all, array_has_any, make_array,
-};
+use datafusion_common::ScalarValue;
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+use datafusion_functions_nested::array_has::{ArrayHas, ArrayHasAll, ArrayHasAny};
+use rand::Rng;
+use rand::SeedableRng;
+use rand::rngs::StdRng;
use std::hint::black_box;
+use std::sync::Arc;
+
+const NUM_ROWS: usize = 10000;
+const SEED: u64 = 42;
+const NULL_DENSITY: f64 = 0.1;
+const NEEDLE_SIZE: usize = 3;
// If not explicitly stated, `array` and `array_size` refer to the haystack array.
fn criterion_benchmark(c: &mut Criterion) {
// Test different array sizes
- let array_sizes = vec![1, 10, 100, 1000, 10000];
+ let array_sizes = vec![10, 100, 500];
for &size in &array_sizes {
bench_array_has(c, size);
@@ -39,50 +51,65 @@ fn criterion_benchmark(c: &mut Criterion) {
bench_array_has_strings(c);
bench_array_has_all_strings(c);
bench_array_has_any_strings(c);
-
- // Edge cases
- bench_array_has_edge_cases(c);
}
fn bench_array_has(c: &mut Criterion, array_size: usize) {
let mut group = c.benchmark_group("array_has_i64");
-
- // Benchmark: element found at beginning
- group.bench_with_input(
- BenchmarkId::new("found_at_start", array_size),
- &array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle = lit(0_i64);
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
- },
- );
-
- // Benchmark: element found at end
+ let list_array = create_int64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
+ let config_options = Arc::new(ConfigOptions::default());
+ let return_field: Arc<Field> = Field::new("result", DataType::Boolean, true).into();
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Field::new("arr", list_array.data_type().clone(), false).into(),
+ Field::new("el", DataType::Int64, false).into(),
+ ];
+
+ // Benchmark: element found
+ let args_found = vec![
+ ColumnarValue::Array(list_array.clone()),
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),
+ ];
group.bench_with_input(
- BenchmarkId::new("found_at_end", array_size),
+ BenchmarkId::new("found", array_size),
&array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle = lit((size - 1) as i64);
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
+ |b, _| {
+ let udf = ArrayHas::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_found.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
},
);
// Benchmark: element not found
+ let args_not_found = vec![
+ ColumnarValue::Array(list_array.clone()),
+ ColumnarValue::Scalar(ScalarValue::Int64(Some(-999))),
+ ];
group.bench_with_input(
BenchmarkId::new("not_found", array_size),
&array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle = lit(-1_i64); // Not in array
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
+ |b, _| {
+ let udf = ArrayHas::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_not_found.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
},
);
@@ -91,45 +118,65 @@ fn bench_array_has(c: &mut Criterion, array_size: usize) {
fn bench_array_has_all(c: &mut Criterion, array_size: usize) {
let mut group = c.benchmark_group("array_has_all");
+ let haystack = create_int64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
+ let list_type = haystack.data_type().clone();
+ let config_options = Arc::new(ConfigOptions::default());
+ let return_field: Arc<Field> = Field::new("result", DataType::Boolean, true).into();
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Field::new("haystack", list_type.clone(), false).into(),
+ Field::new("needle", list_type.clone(), false).into(),
+ ];
// Benchmark: all elements found (small needle)
+ let needle_found = create_int64_list_array(NUM_ROWS, NEEDLE_SIZE, 0.0);
+ let args_found = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_found),
+ ];
group.bench_with_input(
BenchmarkId::new("all_found_small_needle", array_size),
&array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_array = make_array(vec![lit(0_i64), lit(1_i64), lit(2_i64)]);
-
- b.iter(|| black_box(array_has_all(list_array.clone(), needle_array.clone())))
+ |b, _| {
+ let udf = ArrayHasAll::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_found.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
},
);
- // Benchmark: all elements found (medium needle - 10% of haystack)
+ // Benchmark: not all found (needle contains elements outside haystack range)
+ let needle_missing =
+ create_int64_list_array_with_offset(NUM_ROWS, NEEDLE_SIZE, array_size as i64);
+ let args_missing = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_missing),
+ ];
group.bench_with_input(
- BenchmarkId::new("all_found_medium_needle", array_size),
+ BenchmarkId::new("not_all_found", array_size),
&array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_size = (size / 10).max(1);
- let needle = (0..needle_size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let needle_array = make_array(needle);
-
- b.iter(|| black_box(array_has_all(list_array.clone(), needle_array.clone())))
- },
- );
-
- // Benchmark: not all found (early exit)
- group.bench_with_input(
- BenchmarkId::new("early_exit", array_size),
- &array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_array = make_array(vec![lit(0_i64), lit(-1_i64)]); // -1 not in array
-
- b.iter(|| black_box(array_has_all(list_array.clone(), needle_array.clone())))
+ |b, _| {
+ let udf = ArrayHasAll::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_missing.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
},
);
@@ -138,43 +185,65 @@ fn bench_array_has_all(c: &mut Criterion, array_size: usize) {
fn bench_array_has_any(c: &mut Criterion, array_size: usize) {
let mut group = c.benchmark_group("array_has_any");
-
- // Benchmark: first element matches (best case)
+ let haystack = create_int64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
+ let list_type = haystack.data_type().clone();
+ let config_options = Arc::new(ConfigOptions::default());
+ let return_field: Arc<Field> = Field::new("result", DataType::Boolean, true).into();
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Field::new("haystack", list_type.clone(), false).into(),
+ Field::new("needle", list_type.clone(), false).into(),
+ ];
+
+ // Benchmark: some elements match
+ let needle_match = create_int64_list_array(NUM_ROWS, NEEDLE_SIZE, 0.0);
+ let args_match = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_match),
+ ];
group.bench_with_input(
- BenchmarkId::new("first_match", array_size),
+ BenchmarkId::new("some_match", array_size),
&array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_array = make_array(vec![lit(0_i64), lit(-1_i64), lit(-2_i64)]);
-
- b.iter(|| black_box(array_has_any(list_array.clone(), needle_array.clone())))
- },
- );
-
- // Benchmark: last element matches (worst case)
- group.bench_with_input(
- BenchmarkId::new("last_match", array_size),
- &array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_array = make_array(vec![lit(-1_i64), lit(-2_i64), lit(0_i64)]);
-
- b.iter(|| black_box(array_has_any(list_array.clone(), needle_array.clone())))
+ |b, _| {
+ let udf = ArrayHasAny::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_match.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
},
);
// Benchmark: no match
+ let needle_no_match =
+ create_int64_list_array_with_offset(NUM_ROWS, NEEDLE_SIZE, array_size as i64);
+ let args_no_match = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_no_match),
+ ];
group.bench_with_input(
BenchmarkId::new("no_match", array_size),
&array_size,
- |b, &size| {
- let array = (0..size).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_array = make_array(vec![lit(-1_i64), lit(-2_i64), lit(-3_i64)]);
-
- b.iter(|| black_box(array_has_any(list_array.clone(), needle_array.clone())))
+ |b, _| {
+ let udf = ArrayHasAny::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_no_match.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
},
);
@@ -183,29 +252,56 @@ fn bench_array_has_any(c: &mut Criterion, array_size: usize) {
fn bench_array_has_strings(c: &mut Criterion) {
let mut group = c.benchmark_group("array_has_strings");
+ let config_options = Arc::new(ConfigOptions::default());
+ let return_field: Arc<Field> = Field::new("result", DataType::Boolean, true).into();
- // Benchmark with string arrays (common use case for tickers, tags, etc.)
- let sizes = vec![10, 100, 1000];
+ let sizes = vec![10, 100, 500];
for &size in &sizes {
- group.bench_with_input(BenchmarkId::new("found", size), &size, |b, &size| {
- let array = (0..size)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle = lit("TICKER0005");
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
+ let list_array = create_string_list_array(NUM_ROWS, size, NULL_DENSITY);
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Field::new("arr", list_array.data_type().clone(), false).into(),
+ Field::new("el", DataType::Utf8, false).into(),
+ ];
+
+ let args_found = vec![
+ ColumnarValue::Array(list_array.clone()),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("value_1".to_string()))),
+ ];
+ group.bench_with_input(BenchmarkId::new("found", size), &size, |b, _| {
+ let udf = ArrayHas::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_found.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
});
- group.bench_with_input(BenchmarkId::new("not_found", size), &size, |b, &size| {
- let array = (0..size)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle = lit("NOTFOUND");
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
+ let args_not_found = vec![
+ ColumnarValue::Array(list_array.clone()),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("NOTFOUND".to_string()))),
+ ];
+ group.bench_with_input(BenchmarkId::new("not_found", size), &size, |b, _| {
+ let udf = ArrayHas::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_not_found.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
});
}
@@ -214,52 +310,61 @@ fn bench_array_has_strings(c: &mut Criterion) {
fn bench_array_has_all_strings(c: &mut Criterion) {
let mut group = c.benchmark_group("array_has_all_strings");
+ let config_options = Arc::new(ConfigOptions::default());
+ let return_field: Arc<Field> = Field::new("result", DataType::Boolean, true).into();
- // Realistic scenario: checking if a portfolio contains certain tickers
- let portfolio_size = 100;
- let check_sizes = vec![1, 3, 5, 10];
-
- for &check_size in &check_sizes {
- group.bench_with_input(
- BenchmarkId::new("all_found", check_size),
- &check_size,
- |b, &check_size| {
- let portfolio = (0..portfolio_size)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- let list_array = make_array(portfolio);
-
- let checking = (0..check_size)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- let needle_array = make_array(checking);
-
- b.iter(|| {
- black_box(array_has_all(list_array.clone(), needle_array.clone()))
- })
- },
- );
-
- group.bench_with_input(
- BenchmarkId::new("some_missing", check_size),
- &check_size,
- |b, &check_size| {
- let portfolio = (0..portfolio_size)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- let list_array = make_array(portfolio);
-
- let mut checking = (0..check_size - 1)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- checking.push(lit("NOTFOUND".to_string()));
- let needle_array = make_array(checking);
-
- b.iter(|| {
- black_box(array_has_all(list_array.clone(), needle_array.clone()))
- })
- },
- );
+ let sizes = vec![10, 100, 500];
+
+ for &size in &sizes {
+ let haystack = create_string_list_array(NUM_ROWS, size, NULL_DENSITY);
+ let list_type = haystack.data_type().clone();
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Field::new("haystack", list_type.clone(), false).into(),
+ Field::new("needle", list_type.clone(), false).into(),
+ ];
+
+ let needle_found = create_string_list_array(NUM_ROWS, NEEDLE_SIZE, 0.0);
+ let args_found = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_found),
+ ];
+ group.bench_with_input(BenchmarkId::new("all_found", size), &size, |b, _| {
+ let udf = ArrayHasAll::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_found.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
+ });
+
+ let needle_missing =
+ create_string_list_array_with_prefix(NUM_ROWS, NEEDLE_SIZE, "missing_");
+ let args_missing = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_missing),
+ ];
+ group.bench_with_input(BenchmarkId::new("not_all_found", size), &size, |b, _| {
+ let udf = ArrayHasAll::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_missing.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
+ });
}
group.finish();
@@ -267,109 +372,180 @@ fn bench_array_has_all_strings(c: &mut Criterion) {
fn bench_array_has_any_strings(c: &mut Criterion) {
let mut group = c.benchmark_group("array_has_any_strings");
+ let config_options = Arc::new(ConfigOptions::default());
+ let return_field: Arc<Field> = Field::new("result", DataType::Boolean, true).into();
- let portfolio_size = 100;
- let check_sizes = vec![1, 3, 5, 10];
-
- for &check_size in &check_sizes {
- group.bench_with_input(
- BenchmarkId::new("first_matches", check_size),
- &check_size,
- |b, &check_size| {
- let portfolio = (0..portfolio_size)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- let list_array = make_array(portfolio);
-
- let mut checking = vec![lit("TICKER0000".to_string())];
- checking.extend((1..check_size).map(|_| lit("NOTFOUND".to_string())));
- let needle_array = make_array(checking);
-
- b.iter(|| {
- black_box(array_has_any(list_array.clone(), needle_array.clone()))
- })
- },
- );
-
- group.bench_with_input(
- BenchmarkId::new("none_match", check_size),
- &check_size,
- |b, &check_size| {
- let portfolio = (0..portfolio_size)
- .map(|i| lit(format!("TICKER{i:04}")))
- .collect::<Vec<_>>();
- let list_array = make_array(portfolio);
-
- let checking = (0..check_size)
- .map(|i| lit(format!("NOTFOUND{i}")))
- .collect::<Vec<_>>();
- let needle_array = make_array(checking);
-
- b.iter(|| {
- black_box(array_has_any(list_array.clone(), needle_array.clone()))
- })
- },
- );
+ let sizes = vec![10, 100, 500];
+
+ for &size in &sizes {
+ let haystack = create_string_list_array(NUM_ROWS, size, NULL_DENSITY);
+ let list_type = haystack.data_type().clone();
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Field::new("haystack", list_type.clone(), false).into(),
+ Field::new("needle", list_type.clone(), false).into(),
+ ];
+
+ let needle_match = create_string_list_array(NUM_ROWS, NEEDLE_SIZE, 0.0);
+ let args_match = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_match),
+ ];
+ group.bench_with_input(BenchmarkId::new("some_match", size), &size, |b, _| {
+ let udf = ArrayHasAny::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_match.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
+ });
+
+ let needle_no_match =
+ create_string_list_array_with_prefix(NUM_ROWS, NEEDLE_SIZE, "missing_");
+ let args_no_match = vec![
+ ColumnarValue::Array(haystack.clone()),
+ ColumnarValue::Array(needle_no_match),
+ ];
+ group.bench_with_input(BenchmarkId::new("no_match", size), &size, |b, _| {
+ let udf = ArrayHasAny::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args_no_match.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ })
+ });
}
group.finish();
}
-fn bench_array_has_edge_cases(c: &mut Criterion) {
- let mut group = c.benchmark_group("array_has_edge_cases");
-
- // Empty array
- group.bench_function("empty_array", |b| {
- let list_array = make_array(vec![]);
- let needle = lit(1_i64);
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
- });
-
- // Single element array - found
- group.bench_function("single_element_found", |b| {
- let list_array = make_array(vec![lit(1_i64)]);
- let needle = lit(1_i64);
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
- });
-
- // Single element array - not found
- group.bench_function("single_element_not_found", |b| {
- let list_array = make_array(vec![lit(1_i64)]);
- let needle = lit(2_i64);
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
- });
-
- // Array with duplicates
- group.bench_function("array_with_duplicates", |b| {
- let array = vec![lit(1_i64); 1000];
- let list_array = make_array(array);
- let needle = lit(1_i64);
-
- b.iter(|| black_box(array_has(list_array.clone(), needle.clone())))
- });
-
- // array_has_all: empty needle
- group.bench_function("array_has_all_empty_needle", |b| {
- let array = (0..1000).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_array = make_array(vec![]);
-
- b.iter(|| black_box(array_has_all(list_array.clone(), needle_array.clone())))
- });
+fn create_int64_list_array(
+ num_rows: usize,
+ array_size: usize,
+ null_density: f64,
+) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let values = (0..num_rows * array_size)
+ .map(|_| {
+ if rng.random::<f64>() < null_density {
+ None
+ } else {
+ Some(rng.random_range(0..array_size as i64))
+ }
+ })
+ .collect::<Int64Array>();
+ let offsets = (0..=num_rows)
+ .map(|i| (i * array_size) as i32)
+ .collect::<Vec<i32>>();
+
+ Arc::new(
+ ListArray::try_new(
+ Arc::new(Field::new("item", DataType::Int64, true)),
+ OffsetBuffer::new(offsets.into()),
+ Arc::new(values),
+ None,
+ )
+ .unwrap(),
+ )
+}
- // array_has_any: empty needle
- group.bench_function("array_has_any_empty_needle", |b| {
- let array = (0..1000).map(|i| lit(i as i64)).collect::<Vec<_>>();
- let list_array = make_array(array);
- let needle_array = make_array(vec![]);
+/// Like `create_int64_list_array` but values are offset so they won't
+/// appear in a standard list array (useful for "not found" benchmarks).
+fn create_int64_list_array_with_offset(
+ num_rows: usize,
+ array_size: usize,
+ offset: i64,
+) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED + 1);
+ let values = (0..num_rows * array_size)
+ .map(|_| Some(rng.random_range(0..array_size as i64) + offset))
+ .collect::<Int64Array>();
+ let offsets = (0..=num_rows)
+ .map(|i| (i * array_size) as i32)
+ .collect::<Vec<i32>>();
+
+ Arc::new(
+ ListArray::try_new(
+ Arc::new(Field::new("item", DataType::Int64, true)),
+ OffsetBuffer::new(offsets.into()),
+ Arc::new(values),
+ None,
+ )
+ .unwrap(),
+ )
+}
- b.iter(|| black_box(array_has_any(list_array.clone(), needle_array.clone())))
- });
+fn create_string_list_array(
+ num_rows: usize,
+ array_size: usize,
+ null_density: f64,
+) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let values = (0..num_rows * array_size)
+ .map(|_| {
+ if rng.random::<f64>() < null_density {
+ None
+ } else {
+ let idx = rng.random_range(0..array_size);
+ Some(format!("value_{idx}"))
+ }
+ })
+ .collect::<StringArray>();
+ let offsets = (0..=num_rows)
+ .map(|i| (i * array_size) as i32)
+ .collect::<Vec<i32>>();
+
+ Arc::new(
+ ListArray::try_new(
+ Arc::new(Field::new("item", DataType::Utf8, true)),
+ OffsetBuffer::new(offsets.into()),
+ Arc::new(values),
+ None,
+ )
+ .unwrap(),
+ )
+}
- group.finish();
+/// Like `create_string_list_array` but values use a different prefix so
+/// they won't appear in a standard string list array.
+fn create_string_list_array_with_prefix(
+ num_rows: usize,
+ array_size: usize,
+ prefix: &str,
+) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED + 1);
+ let values = (0..num_rows * array_size)
+ .map(|_| {
+ let idx = rng.random_range(0..array_size);
+ Some(format!("{prefix}{idx}"))
+ })
+ .collect::<StringArray>();
+ let offsets = (0..=num_rows)
+ .map(|i| (i * array_size) as i32)
+ .collect::<Vec<i32>>();
+
+ Arc::new(
+ ListArray::try_new(
+ Arc::new(Field::new("item", DataType::Utf8, true)),
+ OffsetBuffer::new(offsets.into()),
+ Arc::new(values),
+ None,
+ )
+ .unwrap(),
+ )
}
criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs
index abc0e7406b..e34239ed49 100644
--- a/datafusion/functions-nested/src/array_has.rs
+++ b/datafusion/functions-nested/src/array_has.rs
@@ -17,7 +17,7 @@
//! [`ScalarUDFImpl`] definitions for array_has, array_has_all and array_has_any functions.
-use arrow::array::{Array, ArrayRef, BooleanArray, Datum, Scalar};
+use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, Datum, Scalar};
use arrow::buffer::BooleanBuffer;
use arrow::datatypes::DataType;
use arrow::row::{RowConverter, Rows, SortField};
@@ -353,36 +353,47 @@ fn array_has_dispatch_for_scalar(
)));
}
let eq_array = compare_with_eq(values, needle, is_nested)?;
- let mut final_contained = vec![None; haystack.len()];
- // Check validity buffer to distinguish between null and empty arrays
+ // When a haystack element is null, `eq()` returns null (not false).
+ // In Arrow, a null BooleanArray entry has validity=0 but an
+ // undefined value bit that may happen to be 1. Since set_indices()
+ // operates on the raw value buffer and ignores validity, we AND the
+ // values with the validity bitmap to clear any undefined bits at
+ // null positions. This ensures set_indices() only yields positions
+ // where the comparison genuinely returned true.
+ let eq_bits = match eq_array.nulls() {
+ Some(nulls) => eq_array.values() & nulls.inner(),
+ None => eq_array.values().clone(),
+ };
+
let validity = match &haystack {
ArrayWrapper::FixedSizeList(arr) => arr.nulls(),
ArrayWrapper::List(arr) => arr.nulls(),
ArrayWrapper::LargeList(arr) => arr.nulls(),
};
+ let mut matches = eq_bits.set_indices().peekable();
+ let mut values = BooleanBufferBuilder::new(haystack.len());
+ values.append_n(haystack.len(), false);
- for (i, (start, end)) in haystack.offsets().tuple_windows().enumerate() {
- let length = end - start;
+ for (i, (_start, end)) in haystack.offsets().tuple_windows().enumerate() {
+ let has_match = matches.peek().is_some_and(|&p| p < end);
- // Check if the array at this position is null
- if let Some(validity_buffer) = validity
- && !validity_buffer.is_valid(i)
- {
- final_contained[i] = None; // null array -> null result
- continue;
+ // Advance past all match positions in this row's range.
+ while matches.peek().is_some_and(|&p| p < end) {
+ matches.next();
}
- // For non-null arrays: length is 0 for empty arrays
- if length == 0 {
- final_contained[i] = Some(false); // empty array -> false
- } else {
- let sliced_array = eq_array.slice(start, length);
- final_contained[i] = Some(sliced_array.true_count() > 0);
+ if has_match && validity.is_none_or(|v| v.is_valid(i)) {
+ values.set_bit(i, true);
}
}
- Ok(Arc::new(BooleanArray::from(final_contained)))
+ // A null haystack row always produces a null output, so we can
+ // reuse the haystack's null buffer directly.
+ Ok(Arc::new(BooleanArray::new(
+ values.finish(),
+ validity.cloned(),
+ )))
}
fn array_has_all_inner(args: &[ArrayRef]) -> Result<ArrayRef> {