Apache DataFusion SQL Query Engine
Remove unnecessary Mutex in SharedMemoryReservation (#21899)
Gabriel61fe692766b6
diff --git a/datafusion/physical-plan/src/common.rs b/datafusion/physical-plan/src/common.rsindex 590f6f09e8..5551685427 100644--- a/datafusion/physical-plan/src/common.rs+++ b/datafusion/physical-plan/src/common.rs@@ -33,10 +33,9 @@ use datafusion_common::{Result, plan_err};use datafusion_execution::memory_pool::MemoryReservation;use futures::{StreamExt, TryStreamExt};-use parking_lot::Mutex;/// [`MemoryReservation`] used across query execution streams-pub(crate) type SharedMemoryReservation = Arc<Mutex<MemoryReservation>>;+pub(crate) type SharedMemoryReservation = Arc<MemoryReservation>;/// Create a vector of record batches from a streampub async fn collect(stream: SendableRecordBatchStream) -> Result<Vec<RecordBatch>> {diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex dbfdf94426..11e036434e 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -82,7 +82,6 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequiremenuse datafusion_common::hash_utils::RandomState;use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::{Stream, StreamExt, ready};-use parking_lot::Mutex;const HASHMAP_SHRINK_SCALE_FACTOR: usize = 4;@@ -549,12 +548,12 @@ impl ExecutionPlan for SymmetricHashJoinExec {let enforce_batch_size_in_joins =context.session_config().enforce_batch_size_in_joins();- let reservation = Arc::new(Mutex::new(+ let reservation = Arc::new(MemoryConsumer::new(format!("SymmetricHashJoinStream[{partition}]")).register(context.memory_pool()),- ));+ );if let Some(g) = graph.as_ref() {- reservation.lock().try_grow(g.size())?;+ reservation.try_grow(g.size())?;}if enforce_batch_size_in_joins {@@ -1745,7 +1744,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {let result = combine_two_batches(&self.schema, equal_result, anti_result)?;let capacity = self.size();self.metrics.stream_memory_usage.set(capacity);- self.reservation.lock().try_resize(capacity)?;+ self.reservation.try_resize(capacity)?;Ok(result)}}diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex d440636050..f5d5d6dbd0 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -322,11 +322,11 @@ impl RepartitionExecState {let mut channels = HashMap::with_capacity(txs.len());for (partition, (tx, rx)) in txs.into_iter().zip(rxs).enumerate() {- let reservation = Arc::new(Mutex::new(+ let reservation = Arc::new(MemoryConsumer::new(format!("{name}[{partition}]")).with_can_spill(true).register(context.memory_pool()),- ));+ );// Create spill channels based on mode:// - preserve_order: one spill channel per (input, output) pair for proper FIFO ordering@@ -1401,7 +1401,7 @@ impl RepartitionExec {// if there is still a receiver, send to itif let Some(channel) = output_channels.get_mut(&partition) {let (batch_to_send, is_memory_batch) =- match channel.reservation.lock().try_grow(size) {+ match channel.reservation.try_grow(size) {Ok(_) => {// Memory available - send in-memory batch(RepartitionBatch::Memory(batch), true)@@ -1419,7 +1419,7 @@ impl RepartitionExec {// If the other end has hung up, it was an early shutdown (e.g. LIMIT)// Only shrink memory if it was a memory batchif is_memory_batch {- channel.reservation.lock().shrink(size);+ channel.reservation.shrink(size);}output_channels.remove(&partition);}@@ -1638,9 +1638,7 @@ impl PerPartitionStream {Some(Some(v)) => match v {Ok(RepartitionBatch::Memory(batch)) => {// Release memory and return batch- self.reservation- .lock()- .shrink(batch.get_array_memory_size());+ self.reservation.shrink(batch.get_array_memory_size());return Poll::Ready(Some(Ok(batch)));}Ok(RepartitionBatch::Spilled) => {