Apache DataFusion SQL Query Engine
Avoid skew in Roundrobin repartition (#18880)
Daniël Heresc2ba08717e06
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex 49dc5b8456..7034b71fd5 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -878,9 +878,9 @@ mod test {partition_row_counts.push(total_rows);}assert_eq!(partition_row_counts.len(), 3);- assert_eq!(partition_row_counts[0], 2);+ assert_eq!(partition_row_counts[0], 1);assert_eq!(partition_row_counts[1], 2);- assert_eq!(partition_row_counts[2], 0);+ assert_eq!(partition_row_counts[2], 1);Ok(())}diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex f665a4fb5a..843d975c7d 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -382,6 +382,9 @@ impl RepartitionExecState {txs,partitioning.clone(),metrics,+ // preserve_order depends on partition index to start from 0+ if preserve_order { 0 } else { i },+ num_input_partitions,));// In a separate task, wait for each input to be done@@ -428,12 +431,19 @@ impl BatchPartitioner {/// Create a new [`BatchPartitioner`] with the provided [`Partitioning`]////// The time spent repartitioning will be recorded to `timer`- pub fn try_new(partitioning: Partitioning, timer: metrics::Time) -> Result<Self> {+ pub fn try_new(+ partitioning: Partitioning,+ timer: metrics::Time,+ input_partition: usize,+ num_input_partitions: usize,+ ) -> Result<Self> {let state = match partitioning {Partitioning::RoundRobinBatch(num_partitions) => {BatchPartitionerState::RoundRobin {num_partitions,- next_idx: 0,+ // Distribute starting index evenly based on input partition, number of input partitions and number of partitions+ // to avoid they all start at partition 0 and heavily skew on the lower partitions+ next_idx: ((input_partition * num_partitions) / num_input_partitions),}}Partitioning::Hash(exprs, num_partitions) => BatchPartitionerState::Hash {@@ -1199,9 +1209,15 @@ impl RepartitionExec {mut output_channels: HashMap<usize, OutputChannel>,partitioning: Partitioning,metrics: RepartitionMetrics,+ input_partition: usize,+ num_input_partitions: usize,) -> Result<()> {- let mut partitioner =- BatchPartitioner::try_new(partitioning, metrics.repartition_time.clone())?;+ let mut partitioner = BatchPartitioner::try_new(+ partitioning,+ metrics.repartition_time.clone(),+ input_partition,+ num_input_partitions,+ )?;// While there are still outputs to send to, keep pulling inputslet mut batches_until_yield = partitioner.num_partitions();@@ -1865,16 +1881,16 @@ mod tests {// output stream 1 should *not* error and have one of the input batcheslet batches = crate::common::collect(output_stream1).await.unwrap();- assert_snapshot!(batches_to_sort_string(&batches), @r#"- +------------------+- | my_awesome_field |- +------------------+- | baz |- | frob |- | gaz |- | grob |- +------------------+- "#);+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +------------------++ | my_awesome_field |+ +------------------++ | baz |+ | frob |+ | gar |+ | goo |+ +------------------++ ");}#[tokio::test]