Apache DataFusion SQL Query Engine
Call take arrays once per repartitioned input batch (#22159)
Gene Bordegaray1ab146ad6cc1
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 873ba8a5ee..f2842e5f77 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -52,8 +52,7 @@ use datafusion_common::stats::Precision;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::transpose;use datafusion_common::{- ColumnStatistics, DataFusionError, HashMap, assert_or_internal_err,- internal_datafusion_err, internal_err,+ ColumnStatistics, DataFusionError, HashMap, assert_or_internal_err, internal_err,};use datafusion_common::{Result, not_impl_err};use datafusion_common_runtime::SpawnedTask;@@ -681,46 +680,8 @@ impl BatchPartitioner {// Finished building index-arrays for output partitionstimer.done();- // Borrowing partitioner timer to prevent moving `self` to closure- let partitioner_timer = &self.timer;-- let mut partitioned_batches = vec![];- for (partition, p_indices) in indices.iter_mut().enumerate() {- if !p_indices.is_empty() {- let taken_indices = std::mem::take(p_indices);- let indices_array: PrimitiveArray<UInt32Type> =- taken_indices.into();-- // Tracking time required for repartitioned batches construction- let _timer = partitioner_timer.timer();-- // Produce batches based on indices- let columns =- take_arrays(batch.columns(), &indices_array, None)?;-- let mut options = RecordBatchOptions::new();- options = options.with_row_count(Some(indices_array.len()));- let batch = RecordBatch::try_new_with_options(- batch.schema(),- columns,- &options,- )- .unwrap();-- partitioned_batches.push(Ok((partition, batch)));-- // Return the taken vec- let (_, buffer, _) = indices_array.into_parts();- let mut vec =- buffer.into_inner().into_vec::<u32>().map_err(|e| {- internal_datafusion_err!(- "Could not convert buffer to vec: {e:?}"- )- })?;- vec.clear();- *p_indices = vec;- }- }+ let partitioned_batches =+ Self::partition_grouped_take(&batch, indices, &self.timer)?;Box::new(partitioned_batches.into_iter())}@@ -736,6 +697,66 @@ impl BatchPartitioner {BatchPartitionerState::Hash { indices, .. } => indices.len(),}}++ /// Build repartitioned hash output batches using one `take` per input batch.+ ///+ /// The hash router first fills one index vector per output partition. This method+ /// concatenates those index vectors, performs one grouped `take_arrays`, and+ /// then returns each output partition as a slice of the reordered batch.+ ///+ /// For example, given partition indices:+ ///+ /// ```text+ /// partition 0: [2, 5]+ /// partition 1: []+ /// partition 2: [0, 3, 4]+ /// ```+ ///+ /// this method takes rows in `[2, 5, 0, 3, 4]` order once, then returns+ /// `partition 0 = slice(0, 2)` and `partition 2 = slice(2, 3)`.+ fn partition_grouped_take(+ batch: &RecordBatch,+ indices: &mut [Vec<u32>],+ timer: &metrics::Time,+ ) -> Result<Vec<Result<(usize, RecordBatch)>>> {+ let mut partition_ranges = Vec::with_capacity(indices.len());+ let mut reordered_indices = Vec::with_capacity(batch.num_rows());++ for (partition, p_indices) in indices.iter_mut().enumerate() {+ if p_indices.is_empty() {+ continue;+ }++ let start = reordered_indices.len();+ reordered_indices.extend_from_slice(p_indices);+ partition_ranges.push((partition, start, p_indices.len()));+ p_indices.clear();+ }++ if reordered_indices.is_empty() {+ return Ok(vec![]);+ }++ let batches = {+ let _timer = timer.timer();+ let indices_array: PrimitiveArray<UInt32Type> = reordered_indices.into();+ let columns = take_arrays(batch.columns(), &indices_array, None)?;++ let mut options = RecordBatchOptions::new();+ options = options.with_row_count(Some(indices_array.len()));+ let reordered_batch =+ RecordBatch::try_new_with_options(batch.schema(), columns, &options)?;++ partition_ranges+ .into_iter()+ .map(|(partition, start, len)| {+ Ok((partition, reordered_batch.slice(start, len)))+ })+ .collect()+ };++ Ok(batches)+ }}/// Maps `N` input partitions to `M` output partitions based on a