Skip to content

Apache DataFusion SQL Query Engine

chore(spm): extract initialize all parititions helper (#23419)

Raz Luvaton4f63ada24056
diff --git a/datafusion/physical-plan/src/sorts/merge.rs b/datafusion/physical-plan/src/sorts/merge.rs
index 4583d19e91..4117789777 100644
--- a/datafusion/physical-plan/src/sorts/merge.rs
+++ b/datafusion/physical-plan/src/sorts/merge.rs
@@ -236,53 +236,24 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
}
return Poll::Ready(None);
}
+
// Once all partitions have set their corresponding cursors for the loser tree,
// we skip the following block. Until then, this function may be called multiple
// times and can return Poll::Pending if any partition returns Poll::Pending.
-
if self.loser_tree.is_empty() {
- // Manual indexing since we're iterating over the vector and shrinking it in the loop
- let mut idx = 0;
- while idx < self.uninitiated_partitions.len() {
- let partition_idx = self.uninitiated_partitions[idx];
- match self.maybe_poll_stream(cx, partition_idx) {
- Poll::Ready(Err(e)) => {
- self.done = true;
- return Poll::Ready(Some(Err(e)));
- }
- Poll::Pending => {
- // The polled stream is pending which means we're already set up to
- // be woken when necessary
- // Try the next stream
- idx += 1;
- }
- _ => {
- // The polled stream is ready
- // Remove it from uninitiated_partitions
- // Don't bump idx here, since a new element will have taken its
- // place which we'll try in the next loop iteration
- // swap_remove will change the partition poll order, but that shouldn't
- // make a difference since we're waiting for all streams to be ready.
- self.uninitiated_partitions.swap_remove(idx);
- }
- }
- }
-
- if self.uninitiated_partitions.is_empty() {
- // If there are no more uninitiated partitions, set up the loser tree and continue
- // to the next phase.
-
- // Claim the memory for the uninitiated partitions
- self.uninitiated_partitions.shrink_to_fit();
- self.init_loser_tree();
- } else {
- // There are still uninitiated partitions so return pending.
- // We only get here if we've polled all uninitiated streams and at least one of them
- // returned pending itself. That means we will be woken as soon as one of the
- // streams would like to be polled again.
- // There is no need to reschedule ourselves eagerly.
- return Poll::Pending;
- }
+ ready!(self.initialize_all_partitions(cx))?;
+ assert_eq!(
+ self.uninitiated_partitions.len(),
+ 0,
+ "all partitions should be initialized"
+ );
+
+ // If there are no more uninitiated partitions, set up the loser tree and continue
+ // to the next phase.
+
+ // Claim the memory for the uninitiated partitions
+ self.uninitiated_partitions.shrink_to_fit();
+ self.init_loser_tree();
}
// NB timer records time taken on drop, so there are no
@@ -327,6 +298,56 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
}
}
+ /// Initialize all partitions, return `Poll::Pending` if any partition returns `Poll::Pending`
+ ///
+ /// This DOES NOT return `Poll::Pending` as soon as the first uninitiated partition returns `Poll::Pending`
+ /// so we can continue to initialize the remaining partitions
+ fn initialize_all_partitions(&mut self, cx: &mut Context) -> Poll<Result<()>> {
+ assert_eq!(
+ self.loser_tree.len(),
+ 0,
+ "loser tree must be empty when initializing"
+ );
+
+ // Manual indexing since we're iterating over the vector and shrinking it in the loop
+ let mut idx = 0;
+ while idx < self.uninitiated_partitions.len() {
+ let partition_idx = self.uninitiated_partitions[idx];
+ match self.maybe_poll_stream(cx, partition_idx) {
+ Poll::Ready(Err(e)) => {
+ self.done = true;
+ return Poll::Ready(Err(e));
+ }
+ Poll::Pending => {
+ // The polled stream is pending which means we're already set up to
+ // be woken when necessary
+ // Try the next stream
+ idx += 1;
+ }
+ _ => {
+ // The polled stream is ready
+ // Remove it from uninitiated_partitions
+ // Don't bump idx here, since a new element will have taken its
+ // place which we'll try in the next loop iteration
+ // swap_remove will change the partition poll order, but that shouldn't
+ // make a difference since we're waiting for all streams to be ready.
+ self.uninitiated_partitions.swap_remove(idx);
+ }
+ }
+ }
+
+ if self.uninitiated_partitions.is_empty() {
+ Poll::Ready(Ok(()))
+ } else {
+ // There are still uninitiated partitions so return pending.
+ // We only get here if we've polled all uninitiated streams and at least one of them
+ // returned pending itself. That means we will be woken as soon as one of the
+ // streams would like to be polled again.
+ // There is no need to reschedule ourselves eagerly.
+ Poll::Pending
+ }
+ }
+
/// For the given partition, updates the poll count. If the current value is the same
/// of the previous value, it increases the count by 1; otherwise, it is reset as 0.
fn update_poll_count_on_the_same_value(&mut self, partition_idx: usize) {