Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize NULL handling in `array_remove` (#21532)

Neil Conway42d983534612
diff --git a/datafusion/functions-nested/benches/array_remove.rs b/datafusion/functions-nested/benches/array_remove.rs
index a494d32239..6fd153548a 100644
--- a/datafusion/functions-nested/benches/array_remove.rs
+++ b/datafusion/functions-nested/benches/array_remove.rs
@@ -389,7 +389,7 @@ fn create_f64_list_array(
if rng.random::<f64>() < null_density {
None
} else {
- Some(rng.random_range(0.0..array_size as f64))
+ Some(rng.random_range(0..array_size as i64) as f64)
}
})
.collect::<Float64Array>();
diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rs
index 730b4b39ac..c428009425 100644
--- a/datafusion/functions-nested/src/remove.rs
+++ b/datafusion/functions-nested/src/remove.rs
@@ -20,10 +20,10 @@
use crate::utils;
use crate::utils::make_scalar_function;
use arrow::array::{
- Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, NullBufferBuilder,
- OffsetSizeTrait, cast::AsArray, make_array,
+ Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, OffsetSizeTrait,
+ cast::AsArray, make_array,
};
-use arrow::buffer::OffsetBuffer;
+use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::datatypes::{DataType, FieldRef};
use datafusion_common::cast::as_int64_array;
use datafusion_common::utils::ListCoercion;
@@ -386,12 +386,13 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
false,
Capacities::Array(original_data.len()),
);
- let mut valid = NullBufferBuilder::new(list_array.len());
+
+ // Pre-compute combined null bitmap
+ let nulls = NullBuffer::union(list_array.nulls(), element_array.nulls());
for (row_index, offset_window) in list_array.offsets().windows(2).enumerate() {
- if list_array.is_null(row_index) || element_array.is_null(row_index) {
+ if nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)) {
offsets.push(offsets[row_index]);
- valid.append_null();
continue;
}
@@ -414,7 +415,6 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
if num_to_remove == 0 {
mutable.extend(0, start, end);
offsets.push(offsets[row_index] + OffsetSize::usize_as(end - start));
- valid.append_non_null();
continue;
}
@@ -445,7 +445,6 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
}
offsets.push(offsets[row_index] + OffsetSize::usize_as(copied));
- valid.append_non_null();
}
let new_values = make_array(mutable.freeze());
@@ -453,7 +452,7 @@ fn general_remove<OffsetSize: OffsetSizeTrait>(
Arc::clone(list_field),
OffsetBuffer::new(offsets.into()),
new_values,
- valid.finish(),
+ nulls,
)?))
}