Skip to content

Apache DataFusion SQL Query Engine

fix: compact sparse string views in FilterExec coalescing (#26436)

Jay Zhand6fee726fead
diff --git a/datafusion/physical-plan/src/coalesce/mod.rs b/datafusion/physical-plan/src/coalesce/mod.rs
index 70c86368c1..107696cd58 100644
--- a/datafusion/physical-plan/src/coalesce/mod.rs
+++ b/datafusion/physical-plan/src/coalesce/mod.rs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{Array, BooleanArray, RecordBatch};
-use arrow::compute::{BatchCoalescer, prep_null_mask_filter};
-use arrow::datatypes::SchemaRef;
+use arrow::array::{Array, AsArray, BooleanArray, RecordBatch};
+use arrow::compute::{BatchCoalescer, filter_record_batch, prep_null_mask_filter};
+use arrow::datatypes::{DataType, SchemaRef};
use datafusion_common::{Result, assert_or_internal_err};
/// Concatenate multiple [`RecordBatch`]es and apply a limit
@@ -124,7 +124,11 @@ impl LimitedBatchCoalescer {
/// avoiding a separate materialization pass compared to calling
/// [`filter_record_batch`] followed by [`Self::push_batch`].
///
- /// [`filter_record_batch`]: arrow::compute::filter_record_batch
+ /// Batches with `Utf8View` or `BinaryView` columns that reference data
+ /// buffers are filtered first and then pushed with [`Self::push_batch`]:
+ /// arrow's fused path keeps the whole input data buffers alive for the few
+ /// selected strings, while [`Self::push_batch`] copies them into compact
+ /// buffers.
pub fn push_batch_with_filter(
&mut self,
batch: RecordBatch,
@@ -135,6 +139,11 @@ impl LimitedBatchCoalescer {
"LimitedBatchCoalescer: cannot push batch after finish"
);
+ if has_view_data_buffers(&batch) {
+ let batch = filter_record_batch(&batch, filter)?;
+ return self.push_batch(batch);
+ }
+
let Some(fetch) = self.fetch else {
self.inner.push_batch_with_filter(batch, filter)?;
return Ok(PushBatchStatus::Continue);
@@ -191,13 +200,23 @@ impl LimitedBatchCoalescer {
}
}
+/// Returns true if `batch` has a `Utf8View` or `BinaryView` column whose views
+/// reference data buffers (strings longer than 12 bytes).
+fn has_view_data_buffers(batch: &RecordBatch) -> bool {
+ batch.columns().iter().any(|array| match array.data_type() {
+ DataType::Utf8View => !array.as_string_view().data_buffers().is_empty(),
+ DataType::BinaryView => !array.as_binary_view().data_buffers().is_empty(),
+ _ => false,
+ })
+}
+
#[cfg(test)]
mod tests {
use super::*;
use std::ops::Range;
use std::sync::Arc;
- use arrow::array::UInt32Array;
+ use arrow::array::{StringViewArray, UInt32Array};
use arrow::compute::concat_batches;
use arrow::datatypes::{DataType, Field, Schema};
@@ -360,6 +379,40 @@ mod tests {
assert!(coalescer.next_completed_batch().is_none());
}
+ /// A selective filter over long strings must not keep the whole input
+ /// data buffer alive for the few selected rows.
+ #[test]
+ fn test_push_batch_with_filter_compacts_string_views() {
+ let strings =
+ (0..1600).map(|i| format!("a string longer than twelve bytes {i:05}"));
+ let array = StringViewArray::from_iter_values(strings);
+ let input_bytes: usize = array.data_buffers().iter().map(|b| b.capacity()).sum();
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "s",
+ DataType::Utf8View,
+ false,
+ )]));
+ let batch =
+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(array)]).unwrap();
+ // Select every 100th row: 16 of 1600 rows
+ let filter = BooleanArray::from_iter((0..1600).map(|i| Some(i % 100 == 0)));
+
+ let mut coalescer = LimitedBatchCoalescer::new(schema, 100, None);
+ coalescer.push_batch_with_filter(batch, &filter).unwrap();
+ coalescer.finish().unwrap();
+ let output = coalescer.next_completed_batch().unwrap();
+
+ let output = output.column(0).as_string_view();
+ assert_eq!(output.len(), 16);
+ assert_eq!(output.value(1), "a string longer than twelve bytes 00100");
+ let output_bytes: usize =
+ output.data_buffers().iter().map(|b| b.capacity()).sum();
+ assert!(
+ output_bytes * 4 < input_bytes,
+ "output keeps {output_bytes} of {input_bytes} input string bytes"
+ );
+ }
+
/// Test for [`LimitedBatchCoalescer`]
///
/// Pushes the input batches to the coalescer and verifies that the resulting