Skip to content

Apache DataFusion SQL Query Engine

test: add regression coverage for terminal hash aggregate drain (#24853)

Huang Qiweib8c41877ba37
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index fc6a676e95..e6ad514556 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -4352,6 +4352,72 @@ mod tests {
Ok(())
}
+ /// Single and Final hash aggregation share the same terminal drain helper. Single
+ /// exercises it directly from raw input without coupling this test to partial state.
+ #[tokio::test]
+ async fn single_grouped_aggregate_avoids_destructive_terminal_drain() -> Result<()> {
+ const BATCH_SIZE: usize = 2;
+ const NUM_GROUPS: usize = 3;
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int32, false),
+ Field::new("value", DataType::Int32, false),
+ ]));
+ let input_batches = vec![RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3, 1, 2, 3])),
+ Arc::new(Int32Array::from(vec![10, 20, 30, 40, 50, 60])),
+ ],
+ )?];
+ let input =
+ TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;
+ let group_by =
+ PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);
+ let udaf = Arc::new(AggregateUDF::from(NoFirstEmitUdaf::new()));
+ let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(
+ AggregateExprBuilder::new(udaf, vec![col("value", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("no_first_emit(value)")
+ .build()?,
+ )];
+ let aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Single,
+ group_by,
+ aggregates,
+ vec![None],
+ input,
+ Arc::clone(&schema),
+ )?);
+ let task_ctx = new_migrated_hash_ctx(BATCH_SIZE);
+
+ let stream = aggregate.execute_typed(0, &task_ctx)?;
+ assert!(matches!(stream, StreamType::SingleHash(_)));
+
+ let stream: SendableRecordBatchStream = stream.into();
+ let batches = collect(stream).await?;
+ assert!(batches.len() > 1, "expected terminal output to be chunked");
+ assert!(
+ batches.iter().all(|batch| batch.num_rows() <= BATCH_SIZE),
+ "terminal output exceeded the configured batch size"
+ );
+ assert_eq!(
+ batches.iter().map(RecordBatch::num_rows).sum::<usize>(),
+ NUM_GROUPS
+ );
+ assert_snapshot!(batches_to_sort_string(&batches), @r"
+ +-----+----------------------+
+ | key | no_first_emit(value) |
+ +-----+----------------------+
+ | 1 | 2 |
+ | 2 | 2 |
+ | 3 | 2 |
+ +-----+----------------------+
+ ");
+
+ Ok(())
+ }
+
#[tokio::test]
async fn limited_distinct_aggregate_uses_migrated_hash_streams() -> Result<()> {
let schema =
@@ -8278,7 +8344,7 @@ mod tests {
Ok(Arc::new(Int64Array::from(counts)))
}
EmitTo::First(_) => internal_err!(
- "partial grouped aggregate output must materialize with EmitTo::All before slicing"
+ "grouped aggregate terminal output must not use EmitTo::First"
),
}
}