Apache DataFusion SQL Query Engine
Do not require mut in memory reservation methods (#19759)
Gabrielb6f7521752ae
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rsindex 2b8385ac2d..abf068db0c 100644--- a/datafusion-cli/src/exec.rs+++ b/datafusion-cli/src/exec.rs@@ -269,7 +269,7 @@ impl StatementExecutor {let options = task_ctx.session_config().options();// Track memory usage for the query result if it's bounded- let mut reservation =+ let reservation =MemoryConsumer::new("DataFusion-Cli").register(task_ctx.memory_pool());if physical_plan.boundedness().is_unbounded() {diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 712f4389f5..b6c606ff46 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -2186,7 +2186,7 @@ mod tests {// configure with same memory / disk managerlet memory_pool = ctx1.runtime_env().memory_pool.clone();- let mut reservation = MemoryConsumer::new("test").register(&memory_pool);+ let reservation = MemoryConsumer::new("test").register(&memory_pool);reservation.grow(100);let disk_manager = ctx1.runtime_env().disk_manager.clone();diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex 6635c9072d..d59b42ed15 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -1360,7 +1360,7 @@ impl FileSink for ParquetSink {parquet_props.clone(),).await?;- let mut reservation = MemoryConsumer::new(format!("ParquetSink[{path}]"))+ let reservation = MemoryConsumer::new(format!("ParquetSink[{path}]")).register(context.memory_pool());file_write_tasks.spawn(async move {while let Some(batch) = rx.recv().await {@@ -1465,7 +1465,7 @@ impl DataSink for ParquetSink {async fn column_serializer_task(mut rx: Receiver<ArrowLeafColumn>,mut writer: ArrowColumnWriter,- mut reservation: MemoryReservation,+ reservation: MemoryReservation,) -> Result<(ArrowColumnWriter, MemoryReservation)> {while let Some(col) = rx.recv().await {writer.write(&col)?;@@ -1550,7 +1550,7 @@ fn spawn_rg_join_and_finalize_task(rg_rows: usize,pool: &Arc<dyn MemoryPool>,) -> SpawnedTask<RBStreamSerializeResult> {- let mut rg_reservation =+ let rg_reservation =MemoryConsumer::new("ParquetSink(SerializedRowGroupWriter)").register(pool);SpawnedTask::spawn(async move {@@ -1682,12 +1682,12 @@ async fn concatenate_parallel_row_groups(mut object_store_writer: Box<dyn AsyncWrite + Send + Unpin>,pool: Arc<dyn MemoryPool>,) -> Result<ParquetMetaData> {- let mut file_reservation =+ let file_reservation =MemoryConsumer::new("ParquetSink(SerializedFileWriter)").register(&pool);while let Some(task) = serialize_rx.recv().await {let result = task.join_unwind().await;- let (serialized_columns, mut rg_reservation, _cnt) =+ let (serialized_columns, rg_reservation, _cnt) =result.map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;let mut rg_out = parquet_writer.next_row_group()?;diff --git a/datafusion/execution/src/memory_pool/mod.rs b/datafusion/execution/src/memory_pool/mod.rsindex fbf9ce41da..b23eede2a0 100644--- a/datafusion/execution/src/memory_pool/mod.rs+++ b/datafusion/execution/src/memory_pool/mod.rs@@ -18,7 +18,7 @@//! [`MemoryPool`] for memory management during query execution, [`proxy`] for//! help with allocation accounting.-use datafusion_common::{Result, internal_err};+use datafusion_common::{Result, internal_datafusion_err};use std::hash::{Hash, Hasher};use std::{cmp::Ordering, sync::Arc, sync::atomic};@@ -322,7 +322,7 @@ impl MemoryConsumer {pool: Arc::clone(pool),consumer: self,}),- size: 0,+ size: atomic::AtomicUsize::new(0),}}}@@ -351,13 +351,13 @@ impl Drop for SharedRegistration {#[derive(Debug)]pub struct MemoryReservation {registration: Arc<SharedRegistration>,- size: usize,+ size: atomic::AtomicUsize,}impl MemoryReservation {/// Returns the size of this reservation in bytespub fn size(&self) -> usize {- self.size+ self.size.load(atomic::Ordering::Relaxed)}/// Returns [MemoryConsumer] for this [MemoryReservation]@@ -367,8 +367,8 @@ impl MemoryReservation {/// Frees all bytes from this reservation back to the underlying/// pool, returning the number of bytes freed.- pub fn free(&mut self) -> usize {- let size = self.size;+ pub fn free(&self) -> usize {+ let size = self.size.load(atomic::Ordering::Relaxed);if size != 0 {self.shrink(size)}@@ -380,60 +380,62 @@ impl MemoryReservation {/// # Panics////// Panics if `capacity` exceeds [`Self::size`]- pub fn shrink(&mut self, capacity: usize) {- let new_size = self.size.checked_sub(capacity).unwrap();+ pub fn shrink(&self, capacity: usize) {+ self.size.fetch_sub(capacity, atomic::Ordering::Relaxed);self.registration.pool.shrink(self, capacity);- self.size = new_size}/// Tries to free `capacity` bytes from this reservation/// if `capacity` does not exceed [`Self::size`]/// Returns new reservation size/// or error if shrinking capacity is more than allocated size- pub fn try_shrink(&mut self, capacity: usize) -> Result<usize> {- if let Some(new_size) = self.size.checked_sub(capacity) {- self.registration.pool.shrink(self, capacity);- self.size = new_size;- Ok(new_size)- } else {- internal_err!(- "Cannot free the capacity {capacity} out of allocated size {}",- self.size+ pub fn try_shrink(&self, capacity: usize) -> Result<usize> {+ let updated = self.size.fetch_update(+ atomic::Ordering::Relaxed,+ atomic::Ordering::Relaxed,+ |prev| prev.checked_sub(capacity),+ );+ updated.map_err(|_| {+ let prev = self.size.load(atomic::Ordering::Relaxed);+ internal_datafusion_err!(+ "Cannot free the capacity {capacity} out of allocated size {prev}")- }+ })}/// Sets the size of this reservation to `capacity`- pub fn resize(&mut self, capacity: usize) {- match capacity.cmp(&self.size) {- Ordering::Greater => self.grow(capacity - self.size),- Ordering::Less => self.shrink(self.size - capacity),+ pub fn resize(&self, capacity: usize) {+ let size = self.size.load(atomic::Ordering::Relaxed);+ match capacity.cmp(&size) {+ Ordering::Greater => self.grow(capacity - size),+ Ordering::Less => self.shrink(size - capacity),_ => {}}}/// Try to set the size of this reservation to `capacity`- pub fn try_resize(&mut self, capacity: usize) -> Result<()> {- match capacity.cmp(&self.size) {- Ordering::Greater => self.try_grow(capacity - self.size)?,- Ordering::Less => self.shrink(self.size - capacity),+ pub fn try_resize(&self, capacity: usize) -> Result<()> {+ let size = self.size.load(atomic::Ordering::Relaxed);+ match capacity.cmp(&size) {+ Ordering::Greater => self.try_grow(capacity - size)?,+ Ordering::Less => self.shrink(size - capacity),_ => {}};Ok(())}/// Increase the size of this reservation by `capacity` bytes- pub fn grow(&mut self, capacity: usize) {+ pub fn grow(&self, capacity: usize) {self.registration.pool.grow(self, capacity);- self.size += capacity;+ self.size.fetch_add(capacity, atomic::Ordering::Relaxed);}/// Try to increase the size of this reservation by `capacity`/// bytes, returning error if there is insufficient capacity left/// in the pool.- pub fn try_grow(&mut self, capacity: usize) -> Result<()> {+ pub fn try_grow(&self, capacity: usize) -> Result<()> {self.registration.pool.try_grow(self, capacity)?;- self.size += capacity;+ self.size.fetch_add(capacity, atomic::Ordering::Relaxed);Ok(())}@@ -447,10 +449,16 @@ impl MemoryReservation {/// # Panics////// Panics if `capacity` exceeds [`Self::size`]- pub fn split(&mut self, capacity: usize) -> MemoryReservation {- self.size = self.size.checked_sub(capacity).unwrap();+ pub fn split(&self, capacity: usize) -> MemoryReservation {+ self.size+ .fetch_update(+ atomic::Ordering::Relaxed,+ atomic::Ordering::Relaxed,+ |prev| prev.checked_sub(capacity),+ )+ .unwrap();Self {- size: capacity,+ size: atomic::AtomicUsize::new(capacity),registration: Arc::clone(&self.registration),}}@@ -458,7 +466,7 @@ impl MemoryReservation {/// Returns a new empty [`MemoryReservation`] with the same [`MemoryConsumer`]pub fn new_empty(&self) -> Self {Self {- size: 0,+ size: atomic::AtomicUsize::new(0),registration: Arc::clone(&self.registration),}}@@ -466,7 +474,7 @@ impl MemoryReservation {/// Splits off all the bytes from this [`MemoryReservation`] into/// a new [`MemoryReservation`] with the same [`MemoryConsumer`]pub fn take(&mut self) -> MemoryReservation {- self.split(self.size)+ self.split(self.size.load(atomic::Ordering::Relaxed))}}@@ -492,7 +500,7 @@ mod tests {#[test]fn test_memory_pool_underflow() {let pool = Arc::new(GreedyMemoryPool::new(50)) as _;- let mut a1 = MemoryConsumer::new("a1").register(&pool);+ let a1 = MemoryConsumer::new("a1").register(&pool);assert_eq!(pool.reserved(), 0);a1.grow(100);@@ -507,7 +515,7 @@ mod tests {a1.try_grow(30).unwrap();assert_eq!(pool.reserved(), 30);- let mut a2 = MemoryConsumer::new("a2").register(&pool);+ let a2 = MemoryConsumer::new("a2").register(&pool);a2.try_grow(25).unwrap_err();assert_eq!(pool.reserved(), 30);@@ -521,7 +529,7 @@ mod tests {#[test]fn test_split() {let pool = Arc::new(GreedyMemoryPool::new(50)) as _;- let mut r1 = MemoryConsumer::new("r1").register(&pool);+ let r1 = MemoryConsumer::new("r1").register(&pool);r1.try_grow(20).unwrap();assert_eq!(r1.size(), 20);@@ -542,10 +550,10 @@ mod tests {#[test]fn test_new_empty() {let pool = Arc::new(GreedyMemoryPool::new(50)) as _;- let mut r1 = MemoryConsumer::new("r1").register(&pool);+ let r1 = MemoryConsumer::new("r1").register(&pool);r1.try_grow(20).unwrap();- let mut r2 = r1.new_empty();+ let r2 = r1.new_empty();r2.try_grow(5).unwrap();assert_eq!(r1.size(), 20);@@ -559,7 +567,7 @@ mod tests {let mut r1 = MemoryConsumer::new("r1").register(&pool);r1.try_grow(20).unwrap();- let mut r2 = r1.take();+ let r2 = r1.take();r2.try_grow(5).unwrap();assert_eq!(r1.size(), 0);diff --git a/datafusion/execution/src/memory_pool/pool.rs b/datafusion/execution/src/memory_pool/pool.rsindex bf74b5f6f4..b10270851c 100644--- a/datafusion/execution/src/memory_pool/pool.rs+++ b/datafusion/execution/src/memory_pool/pool.rs@@ -212,7 +212,7 @@ impl MemoryPool for FairSpillPool {.checked_div(state.num_spill).unwrap_or(spill_available);- if reservation.size + additional > available {+ if reservation.size() + additional > available {return Err(insufficient_capacity_err(reservation,additional,@@ -264,7 +264,7 @@ fn insufficient_capacity_err("Failed to allocate additional {} for {} with {} already allocated for this reservation - {} remain available for the total pool",human_readable_size(additional),reservation.registration.consumer.name,- human_readable_size(reservation.size),+ human_readable_size(reservation.size()),human_readable_size(available))}@@ -526,12 +526,12 @@ mod tests {fn test_fair() {let pool = Arc::new(FairSpillPool::new(100)) as _;- let mut r1 = MemoryConsumer::new("unspillable").register(&pool);+ let r1 = MemoryConsumer::new("unspillable").register(&pool);// Can grow beyond capacity of poolr1.grow(2000);assert_eq!(pool.reserved(), 2000);- let mut r2 = MemoryConsumer::new("r2")+ let r2 = MemoryConsumer::new("r2").with_can_spill(true).register(&pool);// Can grow beyond capacity of pool@@ -563,7 +563,7 @@ mod tests {assert_eq!(r2.size(), 10);assert_eq!(pool.reserved(), 30);- let mut r3 = MemoryConsumer::new("r3")+ let r3 = MemoryConsumer::new("r3").with_can_spill(true).register(&pool);@@ -584,7 +584,7 @@ mod tests {r1.free();assert_eq!(pool.reserved(), 80);- let mut r4 = MemoryConsumer::new("s4").register(&pool);+ let r4 = MemoryConsumer::new("s4").register(&pool);let err = r4.try_grow(30).unwrap_err().strip_backtrace();assert_snapshot!(err, @"Resources exhausted: Failed to allocate additional 30.0 B for s4 with 0.0 B already allocated for this reservation - 20.0 B remain available for the total pool");}@@ -601,18 +601,18 @@ mod tests {// Test: use all the different interfaces to change reservation size// set r1=50, using grow and shrink- let mut r1 = MemoryConsumer::new("r1").register(&pool);+ let r1 = MemoryConsumer::new("r1").register(&pool);r1.grow(50);r1.grow(20);r1.shrink(20);// set r2=15 using try_grow- let mut r2 = MemoryConsumer::new("r2").register(&pool);+ let r2 = MemoryConsumer::new("r2").register(&pool);r2.try_grow(15).expect("should succeed in memory allotment for r2");// set r3=20 using try_resize- let mut r3 = MemoryConsumer::new("r3").register(&pool);+ let r3 = MemoryConsumer::new("r3").register(&pool);r3.try_resize(25).expect("should succeed in memory allotment for r3");r3.try_resize(20)@@ -620,12 +620,12 @@ mod tests {// set r4=10// this should not be reported in top 3- let mut r4 = MemoryConsumer::new("r4").register(&pool);+ let r4 = MemoryConsumer::new("r4").register(&pool);r4.grow(10);// Test: reports if new reservation causes error// using the previously set sizes for other consumers- let mut r5 = MemoryConsumer::new("r5").register(&pool);+ let r5 = MemoryConsumer::new("r5").register(&pool);let res = r5.try_grow(150);assert!(res.is_err());let error = res.unwrap_err().strip_backtrace();@@ -650,7 +650,7 @@ mod tests {let same_name = "foo";// Test: see error message when no consumers recorded yet- let mut r0 = MemoryConsumer::new(same_name).register(&pool);+ let r0 = MemoryConsumer::new(same_name).register(&pool);let res = r0.try_grow(150);assert!(res.is_err());let error = res.unwrap_err().strip_backtrace();@@ -665,7 +665,7 @@ mod tests {r0.grow(10); // make r0=10, pool available=90let new_consumer_same_name = MemoryConsumer::new(same_name);- let mut r1 = new_consumer_same_name.register(&pool);+ let r1 = new_consumer_same_name.register(&pool);// TODO: the insufficient_capacity_err() message is per reservation, not per consumer.// a followup PR will clarify this message "0 bytes already allocated for this reservation"let res = r1.try_grow(150);@@ -695,7 +695,7 @@ mod tests {// will be recognized as different in the TrackConsumersPoollet consumer_with_same_name_but_different_hash =MemoryConsumer::new(same_name).with_can_spill(true);- let mut r2 = consumer_with_same_name_but_different_hash.register(&pool);+ let r2 = consumer_with_same_name_but_different_hash.register(&pool);let res = r2.try_grow(150);assert!(res.is_err());let error = res.unwrap_err().strip_backtrace();@@ -714,10 +714,10 @@ mod tests {// Baseline: see the 2 memory consumerslet setting = make_settings();let _bound = setting.bind_to_scope();- let mut r0 = MemoryConsumer::new("r0").register(&pool);+ let r0 = MemoryConsumer::new("r0").register(&pool);r0.grow(10);let r1_consumer = MemoryConsumer::new("r1");- let mut r1 = r1_consumer.register(&pool);+ let r1 = r1_consumer.register(&pool);r1.grow(20);let res = r0.try_grow(150);@@ -791,13 +791,13 @@ mod tests {.downcast::<TrackConsumersPool<GreedyMemoryPool>>().unwrap();// set r1=20- let mut r1 = MemoryConsumer::new("r1").register(&pool);+ let r1 = MemoryConsumer::new("r1").register(&pool);r1.grow(20);// set r2=15- let mut r2 = MemoryConsumer::new("r2").register(&pool);+ let r2 = MemoryConsumer::new("r2").register(&pool);r2.grow(15);// set r3=45- let mut r3 = MemoryConsumer::new("r3").register(&pool);+ let r3 = MemoryConsumer::new("r3").register(&pool);r3.grow(45);let downcasted = upcasteddiff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex 4f32b6176e..7ada14be66 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -206,7 +206,7 @@ async fn load_left_input(let (batches, _metrics, reservation) = stream.try_fold((Vec::new(), metrics, reservation),- |(mut batches, metrics, mut reservation), batch| async {+ |(mut batches, metrics, reservation), batch| async {let batch_size = batch.get_array_memory_size();// Reserve memory for incoming batchreservation.try_grow(batch_size)?;diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex 44637321a7..b57f913225 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -682,10 +682,10 @@ async fn collect_left_input(let schema = stream.schema();// Load all batches and count the rows- let (batches, metrics, mut reservation) = stream+ let (batches, metrics, reservation) = stream.try_fold((Vec::new(), join_metrics, reservation),- |(mut batches, metrics, mut reservation), batch| async {+ |(mut batches, metrics, reservation), batch| async {let batch_size = batch.get_array_memory_size();// Reserve memory for incoming batchreservation.try_grow(batch_size)?;diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rsindex 508be2e398..d7ece845e9 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs@@ -620,7 +620,7 @@ async fn build_buffered_data(// Combine batches and record number of rowslet initial = (Vec::new(), 0, metrics, reservation);- let (batches, num_rows, metrics, mut reservation) = buffered+ let (batches, num_rows, metrics, reservation) = buffered.try_fold(initial, |mut acc, batch| async {let batch_size = get_record_batch_memory_size(&batch);acc.3.try_grow(batch_size)?;diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 3e8fdf1f3e..a8361f7b29 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -709,7 +709,7 @@ impl ExternalSorter {&self,batch: RecordBatch,metrics: &BaselineMetrics,- mut reservation: MemoryReservation,+ reservation: MemoryReservation,) -> Result<SendableRecordBatchStream> {assert_eq!(get_reserved_bytes_for_record_batch(&batch)?,@@ -736,7 +736,7 @@ impl ExternalSorter {.then({move |batches| async move {match batches {- Ok((schema, sorted_batches, mut reservation)) => {+ Ok((schema, sorted_batches, reservation)) => {// Calculate the total size of sorted batcheslet total_sorted_size: usize = sorted_batches.iter()diff --git a/datafusion/physical-plan/src/sorts/stream.rs b/datafusion/physical-plan/src/sorts/stream.rsindex a510f44e4f..779511a865 100644--- a/datafusion/physical-plan/src/sorts/stream.rs+++ b/datafusion/physical-plan/src/sorts/stream.rs@@ -180,7 +180,7 @@ impl RowCursorStream {self.rows.save(stream_idx, &rows);// track the memory in the newly created Rows.- let mut rows_reservation = self.reservation.new_empty();+ let rows_reservation = self.reservation.new_empty();rows_reservation.try_grow(rows.size())?;Ok(RowValues::new(rows, rows_reservation))}@@ -246,7 +246,7 @@ impl<T: CursorArray> FieldCursorStream<T> {let array = value.into_array(batch.num_rows())?;let size_in_mem = array.get_buffer_memory_size();let array = array.as_any().downcast_ref::<T>().expect("field values");- let mut array_reservation = self.reservation.new_empty();+ let array_reservation = self.reservation.new_empty();array_reservation.try_grow(size_in_mem)?;Ok(ArrayValues::new(self.sort.options,diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rsindex 80c2233d05..4b7e707fcc 100644--- a/datafusion/physical-plan/src/stream.rs+++ b/datafusion/physical-plan/src/stream.rs@@ -1005,7 +1005,7 @@ mod test {.build_arc().unwrap();- let mut reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);+ let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));@@ -1071,7 +1071,7 @@ mod test {.build_arc().unwrap();- let mut reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);+ let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rsindex f1b9e3e88d..1313909adb 100644--- a/datafusion/physical-plan/src/work_table.rs+++ b/datafusion/physical-plan/src/work_table.rs@@ -283,7 +283,7 @@ mod tests {assert!(work_table.take().is_err());let pool = Arc::new(UnboundedMemoryPool::default()) as _;- let mut reservation = MemoryConsumer::new("test_work_table").register(&pool);+ let reservation = MemoryConsumer::new("test_work_table").register(&pool);// Update batch to work_tablelet array: ArrayRef = Arc::new((0..5).collect::<Int32Array>());