Apache DataFusion SQL Query Engine
test: skip count distinct spill memory test under forced hash collisions (#24918)
Raz Luvaton4a93adee064f
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rsindex 39be33587f..9172e34a87 100644--- a/datafusion/core/tests/memory_limit/mod.rs+++ b/datafusion/core/tests/memory_limit/mod.rs@@ -26,14 +26,11 @@ mod nlj_spill_unmatched;mod repartition_mem_limit;mod union_nullable_spill;mod view_spill_compaction;-use arrow::array::{- ArrayRef, DictionaryArray, Int32Array, Int64Array, RecordBatch, StringViewArray,-};+use arrow::array::{ArrayRef, DictionaryArray, Int32Array, RecordBatch, StringViewArray};use arrow::compute::SortOptions;use arrow::datatypes::{Int32Type, SchemaRef};use arrow_schema::{DataType, Field, Schema};use datafusion::assert_batches_eq;-use datafusion::assert_batches_sorted_eq;use datafusion::config::SpillCompression;use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::source::DataSourceExec;@@ -128,83 +125,99 @@ async fn group_by_hash() {.await}-/// `count(distinct)` over integers under a memory limit.-///-/// The integer distinct-count groups accumulator reports the capacity of its-/// buffers in `size()`. After an aggregate stream emits all groups, either to-/// emit partial state early or to spill, it resizes its reservation to the-/// table's reported size and expects it to have shrunk. If the accumulator-/// keeps its capacity, that resize is a grow against an exhausted pool and the-/// query fails although everything was already written out.-const COUNT_DISTINCT_ROWS: usize = 200_000;-const COUNT_DISTINCT_GROUPS: i64 = 64;-const COUNT_DISTINCT_BATCH_ROWS: usize = 8_192;--/// Far below the distinct sets (200k values, several megabytes across the-/// partial and final tables), far above the fixed cost of the stages. With the-/// accumulator releasing its buffers the query passes from 2 MB upwards;-/// without, it fails up to 4 MB with "Decreasing allocation after spilling-/// should succeed" in the final stage or a failed emit in the partial stage.-const COUNT_DISTINCT_MEMORY_LIMIT: usize = 4 * 1024 * 1024;--/// `g` has 64 groups, `v` is unique, so every group holds 3125 distinct values.-fn count_distinct_table() -> MemTable {- let schema = Arc::new(Schema::new(vec![- Field::new("g", DataType::Int64, false),- Field::new("v", DataType::Int64, false),- ]));- let batches = (0..COUNT_DISTINCT_ROWS)- .step_by(COUNT_DISTINCT_BATCH_ROWS)- .map(|start| {- let rows =- start..(start + COUNT_DISTINCT_BATCH_ROWS).min(COUNT_DISTINCT_ROWS);- RecordBatch::try_new(- Arc::clone(&schema),- vec![- Arc::new(Int64Array::from_iter_values(- rows.clone().map(|row| row as i64 % COUNT_DISTINCT_GROUPS),- )),- Arc::new(Int64Array::from_iter_values(rows.map(|row| row as i64))),- ],- )- .unwrap()- })- .collect();- MemTable::try_new(schema, vec![batches]).unwrap()-}--/// Four partial stages emit their state early and four hash-partitioned final-/// stages spill; every one of them must see the accumulator memory drop after-/// emitting all groups.-#[tokio::test]-async fn count_distinct_releases_memory_after_emitting_all() {- let runtime = RuntimeEnvBuilder::new()- .with_memory_limit(COUNT_DISTINCT_MEMORY_LIMIT, 1.0)- .with_disk_manager_builder(DiskManagerBuilder::default())- .build_arc()- .unwrap();- let config = SessionConfig::new().with_target_partitions(4);- let ctx = SessionContext::new_with_config_rt(config, runtime);- ctx.register_table("t", Arc::new(count_distinct_table()))- .unwrap();+/// With `force_hash_collisions` every key hashes alike, so the hash+/// repartitioning sends all groups to a single final stage, whose table then+/// does not fit the limit however well memory is released. The limit is sized+/// for the real distribution across four final stages, so this test is skipped+/// under that feature.+#[cfg(not(feature = "force_hash_collisions"))]+mod count_distinct_spill {+ use super::*;+ use arrow::array::Int64Array;+ use datafusion::assert_batches_sorted_eq;++ /// `count(distinct)` over integers under a memory limit.+ ///+ /// The integer distinct-count groups accumulator reports the capacity of its+ /// buffers in `size()`. After an aggregate stream emits all groups, either to+ /// emit partial state early or to spill, it resizes its reservation to the+ /// table's reported size and expects it to have shrunk. If the accumulator+ /// keeps its capacity, that resize is a grow against an exhausted pool and the+ /// query fails although everything was already written out.+ const COUNT_DISTINCT_ROWS: usize = 200_000;+ const COUNT_DISTINCT_GROUPS: i64 = 64;+ const COUNT_DISTINCT_BATCH_ROWS: usize = 8_192;++ /// Far below the distinct sets (200k values, several megabytes across the+ /// partial and final tables), far above the fixed cost of the stages. With the+ /// accumulator releasing its buffers the query passes from 2 MB upwards;+ /// without, it fails up to 4 MB with "Decreasing allocation after spilling+ /// should succeed" in the final stage or a failed emit in the partial stage.+ const COUNT_DISTINCT_MEMORY_LIMIT: usize = 4 * 1024 * 1024;++ /// `g` has 64 groups, `v` is unique, so every group holds 3125 distinct values.+ fn count_distinct_table() -> MemTable {+ let schema = Arc::new(Schema::new(vec![+ Field::new("g", DataType::Int64, false),+ Field::new("v", DataType::Int64, false),+ ]));+ let batches = (0..COUNT_DISTINCT_ROWS)+ .step_by(COUNT_DISTINCT_BATCH_ROWS)+ .map(|start| {+ let rows =+ start..(start + COUNT_DISTINCT_BATCH_ROWS).min(COUNT_DISTINCT_ROWS);+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from_iter_values(+ rows.clone().map(|row| row as i64 % COUNT_DISTINCT_GROUPS),+ )),+ Arc::new(Int64Array::from_iter_values(+ rows.map(|row| row as i64),+ )),+ ],+ )+ .unwrap()+ })+ .collect();+ MemTable::try_new(schema, vec![batches]).unwrap()+ }- let batches = ctx- .sql("select count(distinct v) as d, count(*) as n from t group by g")- .await- .unwrap()- .collect()- .await- .unwrap_or_else(|error| panic!("query failed under the memory limit: {error}"));-- let per_group = (COUNT_DISTINCT_ROWS as i64 / COUNT_DISTINCT_GROUPS).to_string();- let row = format!("| {per_group} | {per_group} |");- let mut expected = vec!["+------+------+", "| d | n |", "+------+------+"];- expected.extend(std::iter::repeat_n(- row.as_str(),- COUNT_DISTINCT_GROUPS as usize,- ));- expected.push("+------+------+");- assert_batches_sorted_eq!(expected, &batches);+ /// Four partial stages emit their state early and four hash-partitioned final+ /// stages spill; every one of them must see the accumulator memory drop after+ /// emitting all groups.+ #[tokio::test]+ async fn count_distinct_releases_memory_after_emitting_all() {+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(COUNT_DISTINCT_MEMORY_LIMIT, 1.0)+ .with_disk_manager_builder(DiskManagerBuilder::default())+ .build_arc()+ .unwrap();+ let config = SessionConfig::new().with_target_partitions(4);+ let ctx = SessionContext::new_with_config_rt(config, runtime);+ ctx.register_table("t", Arc::new(count_distinct_table()))+ .unwrap();++ let batches = ctx+ .sql("select count(distinct v) as d, count(*) as n from t group by g")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap_or_else(|error| {+ panic!("query failed under the memory limit: {error}")+ });++ let per_group = (COUNT_DISTINCT_ROWS as i64 / COUNT_DISTINCT_GROUPS).to_string();+ let row = format!("| {per_group} | {per_group} |");+ let mut expected = vec!["+------+------+", "| d | n |", "+------+------+"];+ expected.extend(std::iter::repeat_n(+ row.as_str(),+ COUNT_DISTINCT_GROUPS as usize,+ ));+ expected.push("+------+------+");+ assert_batches_sorted_eq!(expected, &batches);+ }}#[tokio::test]