Apache DataFusion SQL Query Engine
perf: batch contiguous extend calls in `array_replace` (#22119)
linfeng484bddbd95d1
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.tomlindex d20542daab..83bed152f6 100644--- a/datafusion/functions-nested/Cargo.toml+++ b/datafusion/functions-nested/Cargo.toml@@ -106,6 +106,10 @@ name = "map"harness = falsename = "array_remove"+[[bench]]+harness = false+name = "array_replace"+[[bench]]harness = falsename = "array_repeat"diff --git a/datafusion/functions-nested/benches/array_replace.rs b/datafusion/functions-nested/benches/array_replace.rsnew file mode 100644index 0000000000..a75b97c3fa--- /dev/null+++ b/datafusion/functions-nested/benches/array_replace.rs@@ -0,0 +1,589 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::{+ Array, ArrayBuilder, ArrayRef, BooleanBuilder, FixedSizeBinaryArray, Int64Builder,+ ListArray, ListBuilder, StringBuilder,+};+use arrow::buffer::{NullBuffer, OffsetBuffer};+use arrow::datatypes::{DataType, Field};+use criterion::{+ criterion_group, criterion_main, {BenchmarkId, Criterion},+};+use datafusion_common::ScalarValue;+use datafusion_common::config::ConfigOptions;+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};+use datafusion_functions_nested::replace::{+ array_replace_all_udf, array_replace_n_udf, array_replace_udf,+};+use rand::Rng;+use rand::SeedableRng;+use rand::rngs::StdRng;+use rand::seq::IndexedRandom;+use std::hint::black_box;+use std::sync::Arc;++// (num_rows, list_size)+const SIZES: &[(usize, usize)] = &[(4_000, 10), (10_000, 100), (10_000, 500)];+const NESTED_SIZES: &[(usize, usize)] = &[(4_000, 10), (3_000, 100), (1_500, 300)];+const SEED: u64 = 42;+const HAYSTACK_NULL_DENSITY: f64 = 0.1;+const NEEDLE_DENSITY: f64 = 0.1;++fn criterion_benchmark(c: &mut Criterion) {+ bench_array_replace_int64(c);+ bench_array_replace_n_int64(c);+ bench_array_replace_all_int64(c);++ bench_array_replace_int64_nested(c);+ bench_array_replace_n_int64_nested(c);+ bench_array_replace_all_int64_nested(c);++ bench_array_replace_strings(c);+ bench_array_replace_boolean(c);+ bench_array_replace_fixed_size_binary(c);+}++fn bench_array_replace_int64(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_int64");++ let filler_values = [None, Some(1), Some(2), Some(3), Some(4), Some(5)];+ let from = 0_i64;+ let to = 6_i64;+ for &(num_rows, list_size) in SIZES {+ let list_array = create_list_array::<Int64Builder, _>(+ num_rows,+ list_size,+ from,+ &filler_values,+ );+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_udf();+ b.iter(|| {+ let args = create_args(+ list_array.clone(),+ ScalarValue::from(from),+ ScalarValue::from(to),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_n_int64(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_n_int64");++ let filler_values = [None, Some(1), Some(2), Some(3), Some(4), Some(5)];+ let from = 0_i64;+ let to = 6_i64;+ for &(num_rows, list_size) in SIZES {+ let list_array = create_list_array::<Int64Builder, _>(+ num_rows,+ list_size,+ from,+ &filler_values,+ );+ let n = (NEEDLE_DENSITY / 2.0 * list_size as f64) as i64;+ let n = 2.max(n);++ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_n_udf();+ b.iter(|| {+ let args = create_args_n(+ list_array.clone(),+ ScalarValue::from(from),+ ScalarValue::from(to),+ ScalarValue::from(n),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_all_int64(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_all_int64");++ let filler_values = [None, Some(1), Some(2), Some(3), Some(4), Some(5)];+ let from = 0_i64;+ let to = 6_i64;+ for &(num_rows, list_size) in SIZES {+ let list_array = create_list_array::<Int64Builder, _>(+ num_rows,+ list_size,+ from,+ &filler_values,+ );+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_all_udf();+ b.iter(|| {+ let args = create_args(+ list_array.clone(),+ ScalarValue::from(from),+ ScalarValue::from(to),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_int64_nested(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_int64_nested");++ let filler_values = [+ None,+ Some(vec![Some(1), Some(0), Some(2), Some(0)]),+ Some(vec![Some(1)]),+ Some(vec![]),+ Some(vec![Some(1), Some(0), Some(2), Some(4), None]),+ Some(vec![None]),+ ];+ let from = vec![Some(1), Some(0), Some(2), Some(4)];+ let to = vec![Some(9), Some(8), Some(7)];+ let from_scalar = list_scalar(&from);+ let to_scalar = list_scalar(&to);+ for &(num_rows, list_size) in NESTED_SIZES {+ let list_array =+ create_nested_i64_list_array(num_rows, list_size, &from, &filler_values);+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_udf();+ b.iter(|| {+ let args = create_args(+ list_array.clone(),+ from_scalar.clone(),+ to_scalar.clone(),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_n_int64_nested(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_n_int64_nested");++ let filler_values = [+ None,+ Some(vec![Some(1), Some(0), Some(2), Some(0)]),+ Some(vec![Some(1)]),+ Some(vec![]),+ Some(vec![Some(1), Some(0), Some(2), Some(4), None]),+ Some(vec![None]),+ ];+ let from = vec![Some(1), Some(0), Some(2), Some(4)];+ let to = vec![Some(9), Some(8), Some(7)];+ let from_scalar = list_scalar(&from);+ let to_scalar = list_scalar(&to);+ for &(num_rows, list_size) in NESTED_SIZES {+ let list_array =+ create_nested_i64_list_array(num_rows, list_size, &from, &filler_values);+ let n = (NEEDLE_DENSITY / 2.0 * list_size as f64) as i64;+ let n = 2.max(n);+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_n_udf();+ b.iter(|| {+ let args = create_args_n(+ list_array.clone(),+ from_scalar.clone(),+ to_scalar.clone(),+ ScalarValue::from(n),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_all_int64_nested(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_all_int64_nested");++ let filler_values = [+ None,+ Some(vec![Some(1), Some(0), Some(2), Some(0)]),+ Some(vec![Some(1)]),+ Some(vec![]),+ Some(vec![Some(1), Some(0), Some(2), Some(4), None]),+ Some(vec![None]),+ ];+ let from = vec![Some(1), Some(0), Some(2), Some(4)];+ let to = vec![Some(9), Some(8), Some(7)];+ let from_scalar = list_scalar(&from);+ let to_scalar = list_scalar(&to);+ for &(num_rows, list_size) in NESTED_SIZES {+ let list_array =+ create_nested_i64_list_array(num_rows, list_size, &from, &filler_values);+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_all_udf();+ b.iter(|| {+ let args = create_args(+ list_array.clone(),+ from_scalar.clone(),+ to_scalar.clone(),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_strings(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_strings");++ let filler_values = [+ None,+ Some("neenee"),+ Some("notthis"),+ Some("value1"),+ Some("abc"),+ Some("hello"),+ ];+ let from = "needle";+ let to = "replacement";+ for &(num_rows, list_size) in SIZES {+ let list_array = create_list_array::<StringBuilder, _>(+ num_rows,+ list_size,+ from,+ &filler_values,+ );+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_udf();+ b.iter(|| {+ let args = create_args(+ list_array.clone(),+ ScalarValue::from(from),+ ScalarValue::from(to),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_boolean(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_boolean");++ let filler_values = [None, Some(false)];+ let from = true;+ let to = false;+ for &(num_rows, list_size) in SIZES {+ let list_array = create_list_array::<BooleanBuilder, _>(+ num_rows,+ list_size,+ from,+ &filler_values,+ );+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_udf();+ b.iter(|| {+ let args = create_args(+ list_array.clone(),+ ScalarValue::from(from),+ ScalarValue::from(to),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++fn bench_array_replace_fixed_size_binary(c: &mut Criterion) {+ let mut group = c.benchmark_group("array_replace_fixed_size_binary");++ const SIZE: usize = 16;+ let filler_values = [+ None,+ Some([2_u8; SIZE]),+ Some([3_u8; SIZE]),+ Some([4_u8; SIZE]),+ Some([5_u8; SIZE]),+ Some([6_u8; SIZE]),+ ];+ let from = [1_u8; SIZE];+ let to = [7_u8; SIZE];+ for &(num_rows, list_size) in SIZES {+ let list_array = create_fixed_size_binary_list_array::<SIZE>(+ num_rows,+ list_size,+ from,+ &filler_values,+ );+ group.bench_with_input(+ BenchmarkId::new(+ "replace",+ format!("list size: {list_size}, num_rows: {num_rows}"),+ ),+ &(list_size, num_rows),+ |b, _| {+ let udf = array_replace_udf();+ b.iter(|| {+ let args = create_args(+ list_array.clone(),+ ScalarValue::FixedSizeBinary(SIZE as i32, Some(from.to_vec())),+ ScalarValue::FixedSizeBinary(SIZE as i32, Some(to.to_vec())),+ );+ black_box(udf.invoke_with_args(args).unwrap())+ })+ },+ );+ }++ group.finish();+}++#[inline]+fn create_args(+ haystack: ArrayRef,+ from: ScalarValue,+ to: ScalarValue,+) -> ScalarFunctionArgs {+ let number_rows = haystack.len();+ let haystack_type = haystack.data_type().clone();+ let from_type = from.data_type().clone();+ let to_type = to.data_type().clone();+ ScalarFunctionArgs {+ args: vec![+ ColumnarValue::Array(haystack),+ ColumnarValue::Scalar(from),+ ColumnarValue::Scalar(to),+ ],+ arg_fields: vec![+ Field::new("haystack", haystack_type.clone(), true).into(),+ Field::new("from", from_type, true).into(),+ Field::new("to", to_type, true).into(),+ ],+ number_rows,+ return_field: Field::new("result", haystack_type, true).into(),+ config_options: Arc::new(ConfigOptions::default()),+ }+}++#[inline]+fn create_args_n(+ haystack: ArrayRef,+ from: ScalarValue,+ to: ScalarValue,+ n: ScalarValue,+) -> ScalarFunctionArgs {+ let number_rows = haystack.len();+ let haystack_type = haystack.data_type().clone();+ let from_type = from.data_type().clone();+ let to_type = to.data_type().clone();+ let n_type = n.data_type().clone();+ ScalarFunctionArgs {+ args: vec![+ ColumnarValue::Array(haystack),+ ColumnarValue::Scalar(from),+ ColumnarValue::Scalar(to),+ ColumnarValue::Scalar(n),+ ],+ arg_fields: vec![+ Field::new("haystack", haystack_type.clone(), true).into(),+ Field::new("from", from_type, true).into(),+ Field::new("to", to_type, true).into(),+ Field::new("n", n_type, true).into(),+ ],+ number_rows,+ return_field: Field::new("result", haystack_type, true).into(),+ config_options: Arc::new(ConfigOptions::default()),+ }+}++fn create_list_array<Builder, Item>(+ num_rows: usize,+ list_size: usize,+ needle_value: Item,+ filler_values: &[Option<Item>],+) -> ArrayRef+where+ Builder: ArrayBuilder + Default + Extend<Option<Item>>,+ Item: Copy,+{+ let mut rng = StdRng::seed_from_u64(SEED);+ let values = (0..num_rows)+ .map(|_| {+ if rng.random_bool(HAYSTACK_NULL_DENSITY) {+ None+ } else {+ let list = (0..list_size)+ .map(|_| {+ if rng.random_bool(NEEDLE_DENSITY) {+ Some(needle_value)+ } else {+ *filler_values.choose(&mut rng).unwrap()+ }+ })+ .collect::<Vec<_>>();+ Some(list)+ }+ })+ .collect::<Vec<_>>();+ Arc::new(ListArray::from_nested_iter::<Builder, _, _, _>(values))+}++fn create_fixed_size_binary_list_array<const SIZE: usize>(+ num_rows: usize,+ list_size: usize,+ needle_value: [u8; SIZE],+ filler_values: &[Option<[u8; SIZE]>],+) -> ArrayRef {+ let mut rng = StdRng::seed_from_u64(SEED);+ let mut buffer = Vec::with_capacity(num_rows * list_size);+ for _ in 0..num_rows {+ for _ in 0..list_size {+ if rng.random_bool(NEEDLE_DENSITY) {+ buffer.push(Some(needle_value));+ } else {+ buffer.push(*filler_values.choose(&mut rng).unwrap());+ }+ }+ }+ let values = FixedSizeBinaryArray::try_from_sparse_iter_with_size(+ buffer.into_iter(),+ SIZE as i32,+ )+ .unwrap();++ let null_buffer = NullBuffer::from_iter(+ (0..num_rows).map(|_| rng.random_bool(1.0 - HAYSTACK_NULL_DENSITY)),+ );++ Arc::new(ListArray::new(+ Field::new("item", DataType::FixedSizeBinary(SIZE as i32), true).into(),+ OffsetBuffer::from_repeated_length(list_size, num_rows),+ Arc::new(values),+ Some(null_buffer),+ ))+}++fn create_nested_i64_list_array(+ num_rows: usize,+ list_size: usize,+ needle_value: &[Option<i64>],+ filler_values: &[Option<Vec<Option<i64>>>],+) -> ArrayRef {+ let mut rng = StdRng::seed_from_u64(SEED);++ let value_builder = Int64Builder::new();+ let inner_builder = ListBuilder::new(value_builder);+ let mut outer_builder = ListBuilder::new(inner_builder);++ for _ in 0..num_rows {+ if rng.random_bool(HAYSTACK_NULL_DENSITY) {+ outer_builder.append(false);+ continue;+ }++ for _ in 0..list_size {+ let inner = outer_builder.values();+ if rng.random_bool(NEEDLE_DENSITY) {+ inner.append_value(needle_value.to_vec());+ } else {+ inner.append_option(filler_values.choose(&mut rng).unwrap().clone());+ }+ }+ outer_builder.append(true);+ }++ Arc::new(outer_builder.finish())+}++fn list_scalar(values: &[Option<i64>]) -> ScalarValue {+ let values = values+ .iter()+ .copied()+ .map(ScalarValue::from)+ .collect::<Vec<_>>();+ ScalarValue::List(ScalarValue::new_list_nullable(&values, &DataType::Int64))+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rsindex 7effd13b69..a9a53a3cb9 100644--- a/datafusion/functions-nested/src/replace.rs+++ b/datafusion/functions-nested/src/replace.rs@@ -347,7 +347,7 @@ fn general_replace<O: OffsetSizeTrait>(let mut counter = 0;// All elements are false, no need to replace, just copy original data- if !eq_array.has_true() {+ if n <= 0 || !eq_array.has_true() {mutable.extend(original_idx.to_usize().unwrap(),start.to_usize().unwrap(),@@ -358,9 +358,18 @@ fn general_replace<O: OffsetSizeTrait>(continue;}+ let mut pending_retain: Option<O> = None;for (i, to_replace) in eq_array.iter().enumerate() {let i = O::usize_as(i);- if let Some(true) = to_replace {+ if to_replace == Some(true) && counter < n {+ // Flush any pending retain run before emitting the replacement.+ if let Some(rs) = pending_retain.take() {+ mutable.extend(+ original_idx.to_usize().unwrap(),+ (start + rs).to_usize().unwrap(),+ (start + i).to_usize().unwrap(),+ );+ }mutable.extend(replace_idx.to_usize().unwrap(), row_index, row_index + 1);counter += 1;if counter == n {@@ -372,16 +381,23 @@ fn general_replace<O: OffsetSizeTrait>();break;}- } else {- // copy original data for false / null matches- mutable.extend(- original_idx.to_usize().unwrap(),- (start + i).to_usize().unwrap(),- (start + i).to_usize().unwrap() + 1,- );+ } else if pending_retain.is_none() {+ pending_retain = Some(i);}}+ // Flush trailing retain run when we exited the loop without ever+ // hitting `counter == n` (i.e. fewer than `n` matches in this row).+ if counter < n+ && let Some(rs) = pending_retain+ {+ mutable.extend(+ original_idx.to_usize().unwrap(),+ (start + rs).to_usize().unwrap(),+ end.to_usize().unwrap(),+ );+ }+offsets.push(offsets[row_index] + (end - start));valid.append_non_null();}diff --git a/datafusion/sqllogictest/test_files/array/array_replace.slt b/datafusion/sqllogictest/test_files/array/array_replace.sltindex 4039915f05..390ed4b946 100644--- a/datafusion/sqllogictest/test_files/array/array_replace.slt+++ b/datafusion/sqllogictest/test_files/array/array_replace.slt@@ -215,21 +215,25 @@ from large_nested_arrays_with_repeating_elements;## array_replace_n (aliases: `list_replace_n`)# array_replace_n scalar function #1-query ???+query ??????selectarray_replace_n(make_array(1, 2, 3, 4), 2, 3, 2),array_replace_n(make_array(1, 4, 4, 5, 4, 6, 7), 4, 0, 2),- array_replace_n(make_array(1, 2, 3), 4, 0, 3);+ array_replace_n(make_array(1, 2, 3), 4, 0, 3),+ array_replace_n(make_array(1, 4, 4), 4, 0, 0),+ array_replace_n(make_array(1, 4, 4), 4, 0, -1),+ array_replace_n(make_array(1, 4, 1, 5), 1, 0, 10);-----[1, 3, 3, 4] [1, 0, 0, 5, 4, 6, 7] [1, 2, 3]+[1, 3, 3, 4] [1, 0, 0, 5, 4, 6, 7] [1, 2, 3] [1, 4, 4] [1, 4, 4] [0, 4, 0, 5]-query ???+query ????selectarray_replace_n(arrow_cast(make_array(1, 2, 3, 4), 'LargeList(Int64)'), 2, 3, 2),array_replace_n(arrow_cast(make_array(1, 4, 4, 5, 4, 6, 7), 'LargeList(Int64)'), 4, 0, 2),- array_replace_n(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 4, 0, 3);+ array_replace_n(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 4, 0, 3),+ array_replace_n(arrow_cast(make_array(1, 4, 4), 'LargeList(Int64)'), 4, 0, 0);-----[1, 3, 3, 4] [1, 0, 0, 5, 4, 6, 7] [1, 2, 3]+[1, 3, 3, 4] [1, 0, 0, 5, 4, 6, 7] [1, 2, 3] [1, 4, 4]query ???select