Apache DataFusion SQL Query Engine
fix: duplicate groups after spilling in legacy hash aggregation with a fallback group key (type not having dedicated impl) (#24889)
Raz Luvaton0015a7521c5f
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rsindex f873f95601..1636b676ba 100644--- a/datafusion/core/tests/memory_limit/mod.rs+++ b/datafusion/core/tests/memory_limit/mod.rs@@ -27,11 +27,12 @@ mod repartition_mem_limit;mod union_nullable_spill;mod view_spill_compaction;use arrow::array::{- ArrayRef, DictionaryArray, Int32Array, Int64Array, RecordBatch, StringArray,- StringViewArray,+ ArrayRef, DictionaryArray, Int32Array, Int64Array, Int64Builder, ListBuilder,+ RecordBatch, StringArray, StringViewArray, StructArray,};+use arrow::buffer::NullBuffer;use arrow::compute::SortOptions;-use arrow::datatypes::{Int32Type, SchemaRef};+use arrow::datatypes::{Fields, Int32Type, SchemaRef};use arrow_schema::{DataType, Field, Schema};use datafusion::assert_batches_eq;use datafusion::config::SpillCompression;@@ -46,6 +47,7 @@ use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_catalog::Session;use datafusion_catalog::streaming::StreamingTable;+use datafusion_common::test_util::batches_to_sort_string;use datafusion_common::{Result, assert_contains};use datafusion_execution::TaskContext;use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};@@ -136,7 +138,6 @@ async fn group_by_hash() {#[cfg(not(feature = "force_hash_collisions"))]mod count_distinct_spill {use super::*;- use arrow::array::Int64Array;use datafusion::assert_batches_sorted_eq;/// `count(distinct)` over integers under a memory limit.@@ -223,6 +224,129 @@ mod count_distinct_spill {}}+/// `GROUP BY` on a single nested key under a memory limit, on both the legacy+/// `GroupedHashAggregateStream` and the migrated streams. The legacy stream used+/// to emit duplicate groups after spilling.+#[tokio::test]+async fn nested_key_spill_keeps_groups_unique() {+ const NESTED_KEY_ROWS: usize = 200_000;+ const NESTED_KEY_GROUPS: i64 = 16;+ const NESTED_KEY_BATCH_ROWS: usize = 8_192;++ /// Small enough that the final stages must spill their `count(distinct)`+ /// state, large enough that the migrated final stream can hold one merged+ /// batch under a `FairSpillPool` shared by four partitions.+ const NESTED_KEY_MEMORY_LIMIT: usize = 8 * 1024 * 1024;++ fn nested_key_struct_fields() -> Fields {+ Fields::from(vec![+ Field::new("list", DataType::new_list(DataType::Int64, true), true),+ Field::new("num", DataType::Int64, true),+ ])+ }++ fn nested_key_table() -> MemTable {+ let schema = Arc::new(Schema::new(vec![+ Field::new_struct("st", nested_key_struct_fields(), true),+ Field::new("v", DataType::Int64, false),+ ]));+ let batches = (0..NESTED_KEY_ROWS)+ .step_by(NESTED_KEY_BATCH_ROWS)+ .map(|start| {+ let rows = start..(start + NESTED_KEY_BATCH_ROWS).min(NESTED_KEY_ROWS);+ let mut list = ListBuilder::new(Int64Builder::new());+ let mut num = Vec::with_capacity(rows.len());+ let mut valid = Vec::with_capacity(rows.len());+ for row in rows.clone() {+ let group = row as i64 % NESTED_KEY_GROUPS;+ match row % 37 {+ 0 => list.append_null(),+ 1 => list.append(true),+ _ => {+ list.values().append_value(group);+ list.values().append_value(group + 1);+ list.append(true);+ }+ }+ num.push((row % 41 != 0).then_some(group));+ valid.push(row % 43 != 0);+ }+ let st = StructArray::new(+ nested_key_struct_fields(),+ vec![Arc::new(list.finish()), Arc::new(Int64Array::from(num))],+ Some(NullBuffer::from(valid)),+ );+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(st),+ Arc::new(Int64Array::from_iter_values(+ rows.map(|row| row as i64),+ )),+ ],+ )+ .unwrap()+ })+ .collect();+ MemTable::try_new(schema, vec![batches]).unwrap()+ }++ async fn run_nested_key_query(memory_limit: Option<usize>, legacy: bool) -> String {+ let mut runtime = RuntimeEnvBuilder::new()+ .with_disk_manager_builder(DiskManagerBuilder::default());+ if let Some(limit) = memory_limit {+ runtime = runtime.with_memory_pool(Arc::new(FairSpillPool::new(limit)));+ }+ let config = SessionConfig::new()+ .with_target_partitions(4)+ // small batches: the merged spill stream arrives in many batches and+ // groups span batch boundaries+ .with_batch_size(64)+ .set_bool("datafusion.execution.enable_migration_aggregate", !legacy);+ let ctx =+ SessionContext::new_with_config_rt(config, runtime.build_arc().unwrap());+ ctx.register_table("t", Arc::new(nested_key_table()))+ .unwrap();++ let df = ctx+ .sql(+ "select st, count(v), count(distinct v), sum(v), avg(v), min(v), max(v) \+ from t group by st",+ )+ .await+ .unwrap();+ let plan = df.create_physical_plan().await.unwrap();++ let task_ctx = ctx.task_ctx();+ let batches = collect_batches(Arc::clone(&plan), task_ctx)+ .await+ .expect("Query execution failed");++ let spill_count = plan_spill_count(plan.as_ref());+ match memory_limit {+ Some(_) => assert_ne!(spill_count, 0, "must have spilled"),+ None => assert_eq!(spill_count, 0, "must not spill on unbounded memory"),+ }++ batches_to_sort_string(&batches)+ }++ let expected = run_nested_key_query(None, false).await;+ assert_eq!(+ run_nested_key_query(None, true).await,+ expected,+ "unbounded, legacy=true"+ );++ for legacy in [true, false] {+ assert_eq!(+ run_nested_key_query(Some(NESTED_KEY_MEMORY_LIMIT), legacy).await,+ expected,+ "spilling, legacy={legacy}"+ );+ }+}+/// A grouped `COUNT(DISTINCT <string>)` gets one accumulator per group, and/// each of those owns a hash set of the distinct values it has seen. Those/// sets used to be created pre-allocated, which costs far more than thediff --git a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rsindex dd77abfeb6..d8f2543996 100644--- a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs+++ b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs@@ -1379,15 +1379,16 @@ impl GroupedHashAggregateStream {// Recreate `group_values` for streaming merge so group ids are assigned// in first-seen order, as required by `GroupOrderingFull`.- // The pre-spill multi-column collector may use `vectorized_intern`, which- // can assign new group ids out of input order under hash collisions.+ // The pre-spill collector may use `vectorized_intern`, which can assign+ // new group ids out of input order under hash collisions. That is the+ // multi-column collector, which also serves a single group column+ // whose type has no specialized single-column collector (for example+ // `Struct` or `Map`), so recreate unconditionally.let group_schema = self.spill_state.merging_group_by.group_schema(&self.spill_state.spill_schema)?;- if group_schema.fields().len() > 1 {- self.group_values = new_group_values(group_schema, &self.group_ordering)?;- }+ self.group_values = new_group_values(group_schema, &self.group_ordering)?;// Use `OutOfMemoryMode::ReportError` from this point on// to ensure we don't spill the spilled data to disk again.