Skip to content

Apache DataFusion SQL Query Engine

fix: reserve initial table memory in aggregate streams (#25900)

Raz Luvaton6924c6a5d45f
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs
index 2cda54d678..2d48ad6790 100644
--- a/datafusion/core/tests/memory_limit/mod.rs
+++ b/datafusion/core/tests/memory_limit/mod.rs
@@ -111,7 +111,10 @@ async fn group_by_row_hash() {
"Resources exhausted: Additional allocation failed",
"for FinalHashAggregateStream[0]",
])
- .with_memory_limit(2_000)
+ // Every aggregate stream reserves its initial table (~4 KB) when created, so
+ // pin the partition count and fit the two partial and two final tables.
+ .with_config(SessionConfig::new().with_target_partitions(2))
+ .with_memory_limit(20_000)
.run()
.await
}
diff --git a/datafusion/physical-plan/src/aggregates/hash_stream.rs b/datafusion/physical-plan/src/aggregates/hash_stream.rs
index 2ef7a6463f..2f1bb9b638 100644
--- a/datafusion/physical-plan/src/aggregates/hash_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/hash_stream.rs
@@ -268,6 +268,9 @@ impl PartialHashAggregateStream {
.with_can_spill(true)
.register(context.memory_pool());
+ // Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
+ reservation.try_grow(hash_table.memory_size())?;
+
Ok(Self {
schema,
input,
@@ -617,6 +620,9 @@ impl FinalHashAggregateStream {
.with_can_spill(can_spill)
.register(context.memory_pool());
+ // Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
+ reservation.try_grow(hash_table.memory_size())?;
+
Ok(Self {
schema,
input,
@@ -926,7 +932,7 @@ mod tests {
// Create constrained memory to trigger early emission but not completely fail
let runtime = RuntimeEnvBuilder::default()
- .with_memory_limit(1024, 1.0) // small enough to start but will trigger pressure
+ .with_memory_limit(5 * 1024, 1.0) // fits the initial table but will trigger pressure
.build_arc()?;
let mut task_ctx = TaskContext::default().with_runtime(runtime);
@@ -1007,7 +1013,7 @@ mod tests {
// Disable skip aggregation so the same input is emitted on memory pressure.
let runtime = RuntimeEnvBuilder::default()
- .with_memory_limit(1024, 1.0)
+ .with_memory_limit(5 * 1024, 1.0)
.build_arc()?;
let session_config = task_ctx.session_config().clone().set(
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 9437f1d83e..aa05b6c5be 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -3361,7 +3361,7 @@ mod tests {
Int64Array, NullArray, StringArray, StructArray, UInt32Array, UInt64Array,
};
use arrow::compute::{SortOptions, concat_batches};
- use arrow::datatypes::{Int32Type, Int64Type};
+ use arrow::datatypes::{Float64Type, Int32Type, Int64Type, UInt32Type};
use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};
use datafusion_common::{DataFusionError, assert_contains, internal_err};
use datafusion_execution::config::SessionConfig;
@@ -3582,6 +3582,14 @@ mod tests {
}
}
+ /// Bytes `aggregate`'s stream for partition 0 reserves when it is created,
+ /// before any input is polled.
+ fn initial_reservation(aggregate: &AggregateExec) -> Result<usize> {
+ let task_ctx = new_migrated_hash_ctx(1);
+ let _stream = aggregate.execute_typed(0, &task_ctx)?;
+ Ok(task_ctx.memory_pool().reserved())
+ }
+
fn new_finite_memory_migrated_hash_ctx(
batch_size: usize,
max_memory: usize,
@@ -4164,13 +4172,6 @@ mod tests {
.build()?,
)];
- let task_ctx = if spill {
- // set to an appropriate value to trigger spill
- new_spill_ctx(2, 1600)
- } else {
- Arc::new(TaskContext::default())
- };
-
let partial_aggregate = Arc::new(AggregateExec::try_new(
AggregateMode::Partial,
grouping_set.clone(),
@@ -4180,6 +4181,13 @@ mod tests {
Arc::clone(&input_schema),
)?);
+ let task_ctx = if spill {
+ // fits the initial table, but not its growth, to trigger spill
+ new_spill_ctx(2, initial_reservation(&partial_aggregate)? + 64)
+ } else {
+ Arc::new(TaskContext::default())
+ };
+
let result =
collect(partial_aggregate.execute(0, Arc::clone(&task_ctx))?).await?;
@@ -4231,8 +4239,9 @@ mod tests {
assert!(final_stats.total_byte_size.get_value().is_some());
let task_ctx = if spill {
- // enlarge memory limit to let the final aggregation finish
- new_spill_ctx(2, 4640)
+ // The fair pool is shared by the partial and final tables, give each
+ // one its initial table plus a little, so the final aggregation spills
+ new_spill_ctx(2, initial_reservation(&merged_aggregate)? + 2 * 64)
} else {
Arc::clone(&task_ctx)
};
@@ -4503,12 +4512,6 @@ mod tests {
let input: Arc<dyn ExecutionPlan> = Arc::new(TestYieldingExec::new(true));
let input_schema = input.schema();
- let runtime = RuntimeEnvBuilder::new()
- .with_memory_limit(1, 1.0)
- .build_arc()?;
- let task_ctx = TaskContext::default().with_runtime(runtime);
- let task_ctx = Arc::new(task_ctx);
-
let groups_none = PhysicalGroupBy::default();
let groups_some = PhysicalGroupBy::new(
vec![(col("a", &input_schema)?, "a".to_string())],
@@ -4543,32 +4546,26 @@ mod tests {
Arc::clone(&input_schema),
)?);
- let stream = partial_aggregate.execute_typed(0, &task_ctx);
+ // fits only the memory reserved when the stream is created
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(initial_reservation(&partial_aggregate)?, 1.0)
+ .build_arc()?;
+ let task_ctx = Arc::new(TaskContext::default().with_runtime(runtime));
+ let stream = partial_aggregate.execute_typed(0, &task_ctx)?;
// ensure that we really got the version we wanted
- let stream = match version {
+ match version {
0 => {
- // the ungrouped stream charges its accumulators' construction-time
- // state up front, so admission fails before any input is read
- let err = stream.err().unwrap();
- assert!(
- matches!(err.find_root(), DataFusionError::ResourcesExhausted(_)),
- "Wrong error type: {err}",
- );
- continue;
+ assert!(matches!(stream, StreamType::AggregateStream(_)));
}
1 => {
- let stream = stream?;
assert!(matches!(stream, StreamType::GroupedHash(_)));
- stream
}
2 => {
- let stream = stream?;
assert!(matches!(stream, StreamType::SingleHash(_)));
- stream
}
_ => panic!("Unknown version: {version}"),
- };
+ }
let stream: SendableRecordBatchStream = stream.into();
let err = collect(stream).await.unwrap_err();
@@ -4682,6 +4679,19 @@ mod tests {
state: vec![0; self.state_bytes],
}))
}
+
+ fn groups_accumulator_supported(&self, _args: AccumulatorArgs) -> bool {
+ true
+ }
+
+ fn create_groups_accumulator(
+ &self,
+ _args: AccumulatorArgs,
+ ) -> Result<Box<dyn GroupsAccumulator>> {
+ Ok(Box::new(ConstructorAllocatingGroupsAccumulator {
+ state: vec![0; self.state_bytes],
+ }))
+ }
}
#[derive(Debug)]
@@ -4711,6 +4721,158 @@ mod tests {
}
}
+ /// Groups accumulator that allocates all of its state at construction. The tests
+ /// using it only create the stream, so the input is never polled.
+ #[derive(Debug)]
+ struct ConstructorAllocatingGroupsAccumulator {
+ state: Vec<u8>,
+ }
+
+ impl GroupsAccumulator for ConstructorAllocatingGroupsAccumulator {
+ fn update_batch(
+ &mut self,
+ _values: &[ArrayRef],
+ _group_indices: &[usize],
+ _opt_filter: Option<&BooleanArray>,
+ _total_num_groups: usize,
+ ) -> Result<()> {
+ internal_err!("input is never polled")
+ }
+
+ fn evaluate(&mut self, _emit_to: EmitTo) -> Result<ArrayRef> {
+ internal_err!("input is never polled")
+ }
+
+ fn state(&mut self, _emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
+ internal_err!("input is never polled")
+ }
+
+ fn merge_batch(
+ &mut self,
+ _values: &[ArrayRef],
+ _group_indices: &[usize],
+ _total_num_groups: usize,
+ ) -> Result<()> {
+ internal_err!("input is never polled")
+ }
+
+ fn convert_to_state(
+ &self,
+ _values: &[ArrayRef],
+ _opt_filter: Option<&BooleanArray>,
+ ) -> Result<Vec<ArrayRef>> {
+ internal_err!("input is never polled")
+ }
+
+ fn size(&self) -> usize {
+ size_of_val(self) + self.state.capacity()
+ }
+ }
+
+ /// Grouped streams hold their hash table for their whole lifetime, so the
+ /// accumulators' construction-time state is charged at `execute`: it is rejected
+ /// before any input is polled and, when it fits, is visible to the pool.
+ #[rstest::rstest]
+ #[case::partial(AggregateMode::Partial, false)]
+ #[case::ordered_partial(AggregateMode::Partial, true)]
+ #[case::partial_reduce(AggregateMode::PartialReduce, false)]
+ #[case::final_stage(AggregateMode::Final, false)]
+ #[case::ordered_final(AggregateMode::Final, true)]
+ #[case::single(AggregateMode::Single, false)]
+ #[case::ordered_single(AggregateMode::Single, true)]
+ fn test_grouped_agg_charges_constructor_allocated_state(
+ #[case] mode: AggregateMode,
+ #[case] ordered: bool,
+ ) -> Result<()> {
+ const STATE_BYTES: usize = 8 * 1024 * 1024;
+ let aggregate =
+ constructor_allocating_grouped_aggregate(mode, ordered, STATE_BYTES)?;
+
+ let too_small = new_finite_memory_migrated_hash_ctx(8, STATE_BYTES / 2)?;
+ let err = aggregate
+ .execute_typed(0, &too_small)
+ .err()
+ .expect("initial accumulator state should not fit the pool");
+ assert!(
+ matches!(err.find_root(), DataFusionError::ResourcesExhausted(_)),
+ "Wrong error type: {err}",
+ );
+
+ let fits = new_finite_memory_migrated_hash_ctx(8, 4 * STATE_BYTES)?;
+ let stream = aggregate.execute_typed(0, &fits)?;
+ match (mode, ordered, &stream) {
+ (AggregateMode::Partial, false, StreamType::PartialHash(_))
+ | (AggregateMode::Partial, true, StreamType::OrderedPartialAggregate(_))
+ | (AggregateMode::PartialReduce, false, StreamType::PartialReduceHash(_))
+ | (AggregateMode::Final, false, StreamType::FinalHash(_))
+ | (AggregateMode::Final, true, StreamType::OrderedFinalAggregate(_))
+ | (AggregateMode::Single, false, StreamType::SingleHash(_))
+ | (AggregateMode::Single, true, StreamType::OrderedSingleAggregate(_)) => {}
+ _ => panic!("unexpected stream for {mode:?}, ordered={ordered}"),
+ }
+ let reserved = fits.memory_pool().reserved();
+ assert!(
+ reserved >= STATE_BYTES,
+ "initial accumulator state not charged: {reserved} bytes reserved",
+ );
+
+ Ok(())
+ }
+
+ /// `constructor_allocating(b) GROUP BY a` over an empty input, sorted on `a` when
+ /// `ordered`. Modes that consume partial state read the partial stage's schema.
+ fn constructor_allocating_grouped_aggregate(
+ mode: AggregateMode,
+ ordered: bool,
+ state_bytes: usize,
+ ) -> Result<AggregateExec> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, true),
+ ]));
+ let group_by =
+ PhysicalGroupBy::new_single(vec![(col("a", &schema)?, "a".to_string())]);
+ let udaf = Arc::new(AggregateUDF::from(ConstructorAllocatingUdaf::new(
+ state_bytes,
+ )));
+ let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(
+ AggregateExprBuilder::new(udaf, vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("constructor_allocating(b)")
+ .build()?,
+ )];
+
+ let empty_input = |schema: SchemaRef| -> Result<Arc<dyn ExecutionPlan>> {
+ let mut input = TestMemoryExec::try_new(&[vec![]], schema, None)?;
+ if ordered {
+ input = input.try_with_sort_information(vec![
+ LexOrdering::new([PhysicalSortExpr::new_default(Arc::new(
+ Column::new("a", 0),
+ ))])
+ .unwrap(),
+ ])?;
+ }
+ Ok(Arc::new(TestMemoryExec::update_cache(&Arc::new(input))))
+ };
+
+ let (input, group_by) = match mode.input_mode() {
+ AggregateInputMode::Raw => (empty_input(Arc::clone(&schema))?, group_by),
+ AggregateInputMode::Partial => {
+ let partial = AggregateExec::try_new(
+ AggregateMode::Partial,
+ group_by.clone(),
+ aggregates.clone(),
+ vec![None],
+ empty_input(Arc::clone(&schema))?,
+ Arc::clone(&schema),
+ )?;
+ (empty_input(partial.schema())?, group_by.as_final())
+ }
+ };
+
+ AggregateExec::try_new(mode, group_by, aggregates, vec![None], input, schema)
+ }
+
#[tokio::test]
async fn partial_grouped_aggregate_uses_raw_partial_stream() -> Result<()> {
let (schema, batches) = some_data();
@@ -5320,14 +5482,21 @@ mod tests {
Ok(())
}
+ /// Partial-reduce aggregate over one input batch of partial states, reducing
+ /// to groups `1, 2, 3` with sums `50, 20, 30`.
fn partial_reduce_test_aggregate() -> Result<AggregateExec> {
- partial_reduce_test_aggregate_with_batches(1)
+ partial_reduce_test_aggregate_with_input(
+ vec![1, 2, 1, 3],
+ vec![10.0, 20.0, 40.0, 30.0],
+ 1,
+ )
}
- /// Partial-reduce aggregate over `num_input_batches` identical input batches
- /// of partial states, each reducing to groups `1, 2, 3` with sums
- /// `50, 20, 30`.
- fn partial_reduce_test_aggregate_with_batches(
+ /// Partial-reduce `SUM(b) GROUP BY a` over `num_input_batches` identical
+ /// input batches of partial states with group keys `a` and partial sums `b`.
+ fn partial_reduce_test_aggregate_with_input(
+ a: Vec<u32>,
+ b: Vec<f64>,
num_input_batches: usize,
) -> Result<AggregateExec> {
let schema = Arc::new(Schema::new(vec![
@@ -5357,8 +5526,8 @@ mod tests {
let partial_state_batch = RecordBatch::try_new(
Arc::clone(&partial_schema),
vec![
- Arc::new(UInt32Array::from(vec![1, 2, 1, 3])),
- Arc::new(Float64Array::from(vec![10.0, 20.0, 40.0, 30.0])),
+ Arc::new(UInt32Array::from(a)),
+ Arc::new(Float64Array::from(b)),
],
)?;
let partial_reduce_input = TestMemoryExec::try_new_exec(
@@ -5404,14 +5573,21 @@ mod tests {
#[tokio::test]
async fn partial_reduce_aggregate_with_memory_limit_emits_early() -> Result<()> {
let num_input_batches = 3;
- let partial_reduce =
- partial_reduce_test_aggregate_with_batches(num_input_batches)?;
+ // Enough groups per batch to outgrow the initial table, even after it is
+ // recreated empty by an early emit.
+ let num_groups = 256;
+ let partial_reduce = partial_reduce_test_aggregate_with_input(
+ (0..num_groups).collect(),
+ (0..num_groups).map(f64::from).collect(),
+ num_input_batches,
+ )?;
+ // Fits only the initial table, so every input batch triggers an early emit.
let runtime = RuntimeEnvBuilder::new()
- .with_memory_limit(1, 1.0)
+ .with_memory_limit(initial_reservation(&partial_reduce)?, 1.0)
.build_arc()?;
// A batch size smaller than the number of flushed groups also covers
// splitting one flush across several output batches.
- let batch_size = 2;
+ let batch_size = 100;
let task_ctx = Arc::new(
TaskContext::default()
.with_session_config(migrated_hash_session_config(batch_size))
@@ -5433,30 +5609,28 @@ mod tests {
num_input_batches
);
- // The table is flushed after every input batch, so each of the three
- // groups is emitted once per input batch instead of being merged into a
- // single row. Each flush is sliced into batches of 2 and 1 rows.
- assert_eq!(output.len(), 2 * num_input_batches);
- assert_snapshot!(batches_to_string(&output), @r"
- +---+-------------+
- | a | SUM(b)[sum] |
- +---+-------------+
- | 1 | 50.0 |
- | 2 | 20.0 |
- | 3 | 30.0 |
- | 1 | 50.0 |
- | 2 | 20.0 |
- | 3 | 30.0 |
- | 1 | 50.0 |
- | 2 | 20.0 |
- | 3 | 30.0 |
- +---+-------------+
- ");
+ // The table is flushed after every input batch, so every group is emitted
+ // once per input batch instead of being merged into a single row. Each
+ // flush of 256 groups is sliced into batches of 100, 100 and 56 rows.
+ let batch_rows: Vec<_> = output.iter().map(RecordBatch::num_rows).collect();
+ assert_eq!(batch_rows, [100, 100, 56].repeat(num_input_batches));
+ let output = concat_batches(&output[0].schema(), &output)?;
+ let a = output.column(0).as_primitive::<UInt32Type>();
+ let sums = output.column(1).as_primitive::<Float64Type>();
+ for row in 0..output.num_rows() {
+ assert_eq!(sums.value(row), f64::from(a.value(row)));
+ }
+ // Each flush contains every group exactly once.
+ for flush in a.values().chunks(num_groups as usize) {
+ let mut keys = flush.to_vec();
+ keys.sort_unstable();
+ assert_eq!(keys, (0..num_groups).collect::<Vec<_>>());
+ }
Ok(())
}
- /// Same shape as [`partial_reduce_test_aggregate_with_batches`], but with multiple
+ /// Same shape as [`partial_reduce_test_aggregate_with_input`], but with multiple
/// group keys.
fn partial_reduce_test_aggregate_rows_multi_group_keys(
num_input_batches: usize,
@@ -5531,8 +5705,9 @@ mod tests {
"expected the Null group column to force the GroupValuesRows fallback"
);
+ // Fits only the initial table, so every input batch triggers an early emit.
let runtime = RuntimeEnvBuilder::new()
- .with_memory_limit(1, 1.0)
+ .with_memory_limit(initial_reservation(&partial_reduce)?, 1.0)
.build_arc()?;
let batch_size = 2;
let task_ctx = Arc::new(
@@ -5965,7 +6140,7 @@ mod tests {
async fn run_first_last_multi_partitions() -> Result<()> {
for is_first_acc in [false, true] {
for spill in [false, true] {
- first_last_multi_partitions(is_first_acc, spill, 5000).await?
+ first_last_multi_partitions(is_first_acc, spill).await?
}
}
Ok(())
@@ -6158,17 +6333,7 @@ mod tests {
//
// and checks whether the function `merge_batch` works correctly for
// FIRST_VALUE and LAST_VALUE functions.
- async fn first_last_multi_partitions(
- is_first_acc: bool,
- spill: bool,
- max_memory: usize,
- ) -> Result<()> {
- let task_ctx = if spill {
- new_spill_ctx(2, max_memory)
- } else {
- Arc::new(TaskContext::default())
- };
-
+ async fn first_last_multi_partitions(is_first_acc: bool, spill: bool) -> Result<()> {
let (schema, data) = some_data_v2();
let partition1 = data[0].clone();
let partition2 = data[1].clone();
@@ -6206,8 +6371,9 @@ mod tests {
memory_exec,
Arc::clone(&schema),
)?);
- let coalesce = Arc::new(CoalescePartitionsExec::new(aggregate_exec))
- as Arc<dyn ExecutionPlan>;
+ let coalesce =
+ Arc::new(CoalescePartitionsExec::new(Arc::clone(&aggregate_exec) as _))
+ as Arc<dyn ExecutionPlan>;
let aggregate_final = Arc::new(AggregateExec::try_new(
AggregateMode::Final,
groups,
@@ -6217,6 +6383,14 @@ mod tests {
schema,
)?) as Arc<dyn ExecutionPlan>;
+ let task_ctx = if spill {
+ // The fair pool is shared by the four partial tables and the final
+ // table, give each one its initial table plus a little
+ new_spill_ctx(2, 5 * (initial_reservation(&aggregate_exec)? + 1000))
+ } else {
+ Arc::new(TaskContext::default())
+ };
+
let result = crate::collect(aggregate_final, task_ctx).await?;
if is_first_acc {
allow_duplicates! {
@@ -7256,7 +7430,9 @@ mod tests {
)?);
let batch_size = 2;
- let memory_pool = Arc::new(FairSpillPool::new(2000));
+ let memory_pool = Arc::new(FairSpillPool::new(
+ initial_reservation(&single_aggregate)? + 200,
+ ));
let task_ctx = Arc::new(
TaskContext::default()
.with_session_config(SessionConfig::new().with_batch_size(batch_size))
@@ -8309,7 +8485,7 @@ mod tests {
// Pool must be large enough for accumulation to start but too small for
// sort_memory after clearing.
- let task_ctx = new_spill_ctx(1, 500);
+ let task_ctx = new_spill_ctx(1, initial_reservation(&aggr)? + 64);
let result = collect(aggr.execute(0, Arc::clone(&task_ctx))?).await;
match &result {
diff --git a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
index 4a70ef2df6..d9f5152456 100644
--- a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
@@ -193,6 +193,10 @@ impl OrderedFinalAggregateStream {
input_order_mode,
metrics,
)?;
+
+ // Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
+ reservation.try_grow(table.memory_size())?;
+
Ok(Self {
reservation,
context: OrderedFinalAggregateContext {
@@ -464,8 +468,8 @@ mod tests {
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
use datafusion_physical_expr::expressions::col;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
- use futures::FutureExt;
use futures::channel::mpsc;
+ use futures::{FutureExt, TryStreamExt};
use std::collections::BTreeMap;
#[derive(Clone, Copy)]
@@ -573,9 +577,7 @@ mod tests {
.build_arc()?,
),
);
- let mut streams = vec![];
- let mut senders = vec![];
- for partition in 0..2 {
+ let new_stream = |partition| -> Result<_> {
let (sender, receiver) = mpsc::unbounded();
let input = Box::pin(RecordBatchStreamAdapter::new(
Arc::clone(&partial_schema),
@@ -588,9 +590,8 @@ mod tests {
input,
aggregate.input_order_mode(),
)?;
- senders.push(sender);
- streams.push(stream.into_stream());
- }
+ Ok((sender, stream.into_stream()))
+ };
let mut expected = BTreeMap::new();
let mut make_batch = |partition: i64, start: i64| {
for value in start..start + 128 {
@@ -632,32 +633,34 @@ mod tests {
// Keep partition 1's incomplete ordered run live while partition 0 spills
// and replays. Channel inputs return Pending after each supplied batch,
// making this interleaving independent of task scheduling.
+ let (second_sender, mut second) = new_stream(1)?;
for batch in 0..55 {
- senders[1]
+ second_sender
.unbounded_send(Ok(make_batch(1, batch * 128)))
.unwrap();
- assert!(streams[1].next().now_or_never().is_none());
+ assert!(second.next().now_or_never().is_none());
}
let held = pool.reserved();
assert!(held > 500 * 1024);
+ // Created after `held` is measured, since the stream reserves its initial table.
+ let (first_sender, mut first) = new_stream(0)?;
for batch in 0..input_batches {
// Repeated keys cross spill runs, so replay must merge their sums.
- senders[0]
+ first_sender
.unbounded_send(Ok(make_batch(0, batch * 128)))
.unwrap();
- assert!(streams[0].next().now_or_never().is_none());
+ assert!(first.next().now_or_never().is_none());
}
if limit == 600 * 1024 {
assert!(aggregate.metrics().unwrap().spill_count().unwrap() > 0);
}
- let mut first = streams.remove(0);
match finish {
Finish::Collect => {
- senders[0].close_channel();
+ first_sender.close_channel();
let mut output = collect(first).await?;
assert_eq!(pool.reserved(), held);
- senders[1].close_channel();
- output.extend(collect(streams.remove(0)).await?);
+ second_sender.close_channel();
+ output.extend(second.by_ref().try_collect::<Vec<_>>().await?);
let mut actual = BTreeMap::new();
for batch in output {
let a = batch
@@ -698,19 +701,19 @@ mod tests {
);
}
Finish::DropDuringMerge => {
- senders[0].close_channel();
+ first_sender.close_channel();
assert!(first.next().now_or_never().is_none());
drop(first);
}
Finish::DropDuringReplay => {
- senders[0].close_channel();
+ first_sender.close_channel();
first.next().await.unwrap()?;
assert!(pool.reserved() > held);
drop(first);
assert_eq!(pool.reserved(), held);
}
Finish::InputError => {
- senders[0]
+ first_sender
.unbounded_send(datafusion_common::exec_err!(
"injected input failure"
))
@@ -721,7 +724,7 @@ mod tests {
drop(first);
}
}
- drop(streams);
+ drop(second);
assert_eq!(pool.reserved(), 0);
Ok(())
}
diff --git a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs
index dd845353c7..31046ec3d6 100644
--- a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs
@@ -171,6 +171,9 @@ impl OrderedPartialAggregateStream {
))
.register(context.memory_pool());
+ // Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
+ reservation.try_grow(table.memory_size())?;
+
Ok(Self {
reservation,
context: OrderedPartialAggregateContext {
diff --git a/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs
index 97afab093a..834fae3cc6 100644
--- a/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs
@@ -193,6 +193,9 @@ impl OrderedSingleAggregateStream {
.with_can_spill(can_spill)
.register(context.memory_pool());
+ // Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
+ reservation.try_grow(table.memory_size())?;
+
Ok(Self {
schema,
input,
diff --git a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs
index d8ea2f711e..d5057d53cf 100644
--- a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs
@@ -209,6 +209,9 @@ impl PartialReduceHashAggregateStream {
.with_can_spill(true)
.register(context.memory_pool());
+ // Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
+ reservation.try_grow(hash_table.memory_size())?;
+
Ok(Self {
schema,
input,
diff --git a/datafusion/physical-plan/src/aggregates/single_stream.rs b/datafusion/physical-plan/src/aggregates/single_stream.rs
index 288bf025e4..f82c90f9dc 100644
--- a/datafusion/physical-plan/src/aggregates/single_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/single_stream.rs
@@ -219,6 +219,9 @@ impl SingleHashAggregateStream {
.with_can_spill(can_spill)
.register(context.memory_pool());
+ // Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
+ reservation.try_grow(hash_table.memory_size())?;
+
Ok(Self {
schema,
input,