Apache DataFusion SQL Query Engine
fix: compact sparse string views in FilterExec coalescing (#26436)
Jay Zhand6fee726fead
diff --git a/datafusion/physical-plan/src/coalesce/mod.rs b/datafusion/physical-plan/src/coalesce/mod.rsindex 70c86368c1..107696cd58 100644--- a/datafusion/physical-plan/src/coalesce/mod.rs+++ b/datafusion/physical-plan/src/coalesce/mod.rs@@ -15,9 +15,9 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{Array, BooleanArray, RecordBatch};-use arrow::compute::{BatchCoalescer, prep_null_mask_filter};-use arrow::datatypes::SchemaRef;+use arrow::array::{Array, AsArray, BooleanArray, RecordBatch};+use arrow::compute::{BatchCoalescer, filter_record_batch, prep_null_mask_filter};+use arrow::datatypes::{DataType, SchemaRef};use datafusion_common::{Result, assert_or_internal_err};/// Concatenate multiple [`RecordBatch`]es and apply a limit@@ -124,7 +124,11 @@ impl LimitedBatchCoalescer {/// avoiding a separate materialization pass compared to calling/// [`filter_record_batch`] followed by [`Self::push_batch`].///- /// [`filter_record_batch`]: arrow::compute::filter_record_batch+ /// Batches with `Utf8View` or `BinaryView` columns that reference data+ /// buffers are filtered first and then pushed with [`Self::push_batch`]:+ /// arrow's fused path keeps the whole input data buffers alive for the few+ /// selected strings, while [`Self::push_batch`] copies them into compact+ /// buffers.pub fn push_batch_with_filter(&mut self,batch: RecordBatch,@@ -135,6 +139,11 @@ impl LimitedBatchCoalescer {"LimitedBatchCoalescer: cannot push batch after finish");+ if has_view_data_buffers(&batch) {+ let batch = filter_record_batch(&batch, filter)?;+ return self.push_batch(batch);+ }+let Some(fetch) = self.fetch else {self.inner.push_batch_with_filter(batch, filter)?;return Ok(PushBatchStatus::Continue);@@ -191,13 +200,23 @@ impl LimitedBatchCoalescer {}}+/// Returns true if `batch` has a `Utf8View` or `BinaryView` column whose views+/// reference data buffers (strings longer than 12 bytes).+fn has_view_data_buffers(batch: &RecordBatch) -> bool {+ batch.columns().iter().any(|array| match array.data_type() {+ DataType::Utf8View => !array.as_string_view().data_buffers().is_empty(),+ DataType::BinaryView => !array.as_binary_view().data_buffers().is_empty(),+ _ => false,+ })+}+#[cfg(test)]mod tests {use super::*;use std::ops::Range;use std::sync::Arc;- use arrow::array::UInt32Array;+ use arrow::array::{StringViewArray, UInt32Array};use arrow::compute::concat_batches;use arrow::datatypes::{DataType, Field, Schema};@@ -360,6 +379,40 @@ mod tests {assert!(coalescer.next_completed_batch().is_none());}+ /// A selective filter over long strings must not keep the whole input+ /// data buffer alive for the few selected rows.+ #[test]+ fn test_push_batch_with_filter_compacts_string_views() {+ let strings =+ (0..1600).map(|i| format!("a string longer than twelve bytes {i:05}"));+ let array = StringViewArray::from_iter_values(strings);+ let input_bytes: usize = array.data_buffers().iter().map(|b| b.capacity()).sum();+ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ DataType::Utf8View,+ false,+ )]));+ let batch =+ RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(array)]).unwrap();+ // Select every 100th row: 16 of 1600 rows+ let filter = BooleanArray::from_iter((0..1600).map(|i| Some(i % 100 == 0)));++ let mut coalescer = LimitedBatchCoalescer::new(schema, 100, None);+ coalescer.push_batch_with_filter(batch, &filter).unwrap();+ coalescer.finish().unwrap();+ let output = coalescer.next_completed_batch().unwrap();++ let output = output.column(0).as_string_view();+ assert_eq!(output.len(), 16);+ assert_eq!(output.value(1), "a string longer than twelve bytes 00100");+ let output_bytes: usize =+ output.data_buffers().iter().map(|b| b.capacity()).sum();+ assert!(+ output_bytes * 4 < input_bytes,+ "output keeps {output_bytes} of {input_bytes} input string bytes"+ );+ }+/// Test for [`LimitedBatchCoalescer`]////// Pushes the input batches to the coalescer and verifies that the resulting