Skip to content

Apache DataFusion SQL Query Engine

chore: mark partial reduce stream as can handle back pressure (spillable) (#25075)

Raz Luvaton35c56b020e19
diff --git a/datafusion/physical-plan/src/aggregates/hash_stream.rs b/datafusion/physical-plan/src/aggregates/hash_stream.rs
index 2195b5eb60..796a2b2885 100644
--- a/datafusion/physical-plan/src/aggregates/hash_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/hash_stream.rs
@@ -420,6 +420,9 @@ impl PartialHashAggregateStream {
let reservation =
MemoryConsumer::new(format!("PartialHashAggregateStream[{partition}]"))
+ // We interpret 'can spill' as 'can handle memory back pressure'.
+ // This value needs to be set to true for the default memory pool implementations
+ // to ensure fair application of back pressure amongst the memory consumers.
.with_can_spill(true)
.register(context.memory_pool());
diff --git a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs
index c9a6cb675d..286a324e96 100644
--- a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs
@@ -198,6 +198,10 @@ impl PartialReduceHashAggregateStream {
let reservation =
MemoryConsumer::new(format!("PartialReduceHashAggregateStream[{partition}]"))
+ // We interpret 'can spill' as 'can handle memory back pressure'.
+ // This value needs to be set to true for the default memory pool implementations
+ // to ensure fair application of back pressure amongst the memory consumers.
+ .with_can_spill(true)
.register(context.memory_pool());
Ok(Self {