Skip to content

Apache DataFusion SQL Query Engine

fix: skip count-distinct byte tests under force_hash_collisions (#25020)

DevShiba6ab4ce660623
diff --git a/datafusion/functions-aggregate-common/Cargo.toml b/datafusion/functions-aggregate-common/Cargo.toml
index 1714e1800a..0898e6b4c5 100644
--- a/datafusion/functions-aggregate-common/Cargo.toml
+++ b/datafusion/functions-aggregate-common/Cargo.toml
@@ -40,6 +40,10 @@ workspace = true
[lib]
name = "datafusion_functions_aggregate_common"
+[features]
+# Used for testing ONLY: causes all values to hash to the same value (test for collisions)
+force_hash_collisions = ["datafusion-common/force_hash_collisions"]
+
[dependencies]
arrow = { workspace = true }
datafusion-common = { workspace = true }
diff --git a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/bytes.rs b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/bytes.rs
index d955d343ad..c230a27c02 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/bytes.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/bytes.rs
@@ -166,7 +166,16 @@ impl Accumulator for BytesViewDistinctCountAccumulator {
}
}
-#[cfg(test)]
+/// With `force_hash_collisions` every distinct value collides into the same
+/// hash bucket, so both `ArrowBytesSet`/`ArrowBytesViewSet` degrade to a
+/// linear scan per insert. These tests insert up to 500,000 distinct values
+/// per accumulator (twice, once lazily constructed and once pre-allocated),
+/// which is O(n) under a real hash and O(n^2) under a forced collision,
+/// turning a sub-second run into a multi-hour one and hanging CI (see
+/// apache/datafusion#25011). Skipped under that feature, matching the
+/// `count_distinct_spill` precedent in
+/// `datafusion/core/tests/memory_limit/mod.rs`.
+#[cfg(all(test, not(feature = "force_hash_collisions")))]
mod tests {
use super::*;
use arrow::array::{StringArray, StringViewArray};