Skip to content

Apache DataFusion SQL Query Engine

test: gate hash-dependent approx_distinct tests behind not(force_hash_collisions) (#23053)

Phoenixb0afa3a0d3d5
diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml
index ff89808f0b..c1b992a6d8 100644
--- a/datafusion/functions-aggregate/Cargo.toml
+++ b/datafusion/functions-aggregate/Cargo.toml
@@ -94,3 +94,6 @@ harness = false
[[bench]]
name = "percentile_cont"
harness = false
+
+[features]
+force_hash_collisions = ["datafusion-common/force_hash_collisions"]
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index 04ae5c1b35..90cc8d0630 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -829,12 +829,104 @@ fn is_hll_groups_type(data_type: &DataType) -> bool {
#[cfg(test)]
mod tests {
use super::*;
- use arrow::array::{AsArray, Int64Array, StringViewArray};
use std::hash::BuildHasher;
- use std::sync::Arc;
- // A string longer than the 12-byte inline limit
- const LONG: &str = "this string is definitely longer than twelve bytes";
+ #[cfg(not(feature = "force_hash_collisions"))]
+ mod real_hash_test {
+ use super::*;
+ use arrow::array::{AsArray, Int64Array, StringViewArray};
+ use std::sync::Arc;
+ // A string longer than the 12-byte inline limit
+ const LONG: &str = "this string is definitely longer than twelve bytes";
+
+ fn distinct_count(acc: &mut HLLAccumulator) -> u64 {
+ match acc.evaluate().unwrap() {
+ ScalarValue::UInt64(Some(v)) => v,
+ other => panic!("unexpected evaluate result: {other:?}"),
+ }
+ }
+
+ /// `approx_distinct(v) FILTER (WHERE nullable_bool)` — a NULL filter row
+ /// must not be counted (null filter is treated the same as false).
+ #[test]
+ fn update_batch_nullable_filter_excludes_null_filter_rows() {
+ let values: ArrayRef = Arc::new(Int64Array::from(vec![1i64, 2, 3, 4, 5]));
+ // row 0: filter=true, row 1: filter=NULL, row 2: filter=false,
+ // row 3: filter=NULL, row 4: filter=true
+ let filter =
+ BooleanArray::from(vec![Some(true), None, Some(false), None, Some(true)]);
+
+ let mut acc = HllGroupsAccumulator::new();
+ // put all rows in group 0
+ let group_indices = vec![0usize; 5];
+ acc.update_batch(&[values], &group_indices, Some(&filter), 1)
+ .unwrap();
+
+ // Only rows 0 and 4 (values 1 and 5) should be counted.
+ let result = acc.evaluate(EmitTo::All).unwrap();
+ let counts = result.as_any().downcast_ref::<UInt64Array>().unwrap();
+ // reference: hash 1 and 5 into a dense sketch
+ let expected = reference_count(&[h(1), h(5)]);
+ assert_eq!(counts.value(0), expected);
+ }
+
+ /// Regression: a short (≤ 12-byte) Utf8View string must hash identically
+ /// in an all-inline batch and in a mixed batch that also contains a long
+ /// string (which forces a data buffer).
+ #[test]
+ fn utf8view_groups_short_string_hashed_consistently_across_batches() {
+ // Batch 1: all-inline (no data buffers) — "aaa" is hashed as u128 view.
+ let batch1: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", "bbb"]));
+ assert!(batch1.as_string_view().data_buffers().is_empty());
+
+ // Batch 2: mixed — LONG forces a data buffer; "aaa" must still be
+ // hashed as u128 view so it matches its appearance in batch 1.
+ let batch2: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", LONG]));
+ assert!(!batch2.as_string_view().data_buffers().is_empty());
+
+ let group_indices = vec![0usize, 0];
+ let mut acc = HllGroupsAccumulator::new();
+ acc.update_batch(&[batch1], &group_indices, None, 1)
+ .unwrap();
+ acc.update_batch(&[batch2], &group_indices, None, 1)
+ .unwrap();
+
+ // True distinct values: {"aaa", "bbb", LONG} == 3.
+ let result = acc.evaluate(EmitTo::All).unwrap();
+ let counts = result.as_any().downcast_ref::<UInt64Array>().unwrap();
+ assert_eq!(counts.value(0), 3);
+ }
+
+ /// Regression: a short (≤ 12-byte) Utf8View string must hash identically
+ /// regardless of which batch it appears in — all-inline or mixed.
+ #[test]
+ fn utf8view_acc_split_batches_match_single_mixed_batch() {
+ // Multiset: {"aaa" x2, "bbb", LONG}, so 3 distinct values.
+ let mixed: ArrayRef =
+ Arc::new(StringViewArray::from(vec!["aaa", "bbb", LONG, "aaa"]));
+ let mut acc_single = HLLAccumulator::new();
+ acc_single.update_batch(&[mixed]).unwrap();
+
+ // Same multiset, but split so "aaa" lands in both an all-inline batch
+ // and a batch with a data buffer (forced by LONG).
+ let inline_only: ArrayRef =
+ Arc::new(StringViewArray::from(vec!["aaa", "bbb"]));
+ let with_buffer: ArrayRef =
+ Arc::new(StringViewArray::from(vec!["aaa", LONG]));
+ assert!(inline_only.as_string_view().data_buffers().is_empty());
+ assert!(!with_buffer.as_string_view().data_buffers().is_empty());
+
+ let mut acc_split = HLLAccumulator::new();
+ acc_split.update_batch(&[inline_only]).unwrap();
+ acc_split.update_batch(&[with_buffer]).unwrap();
+
+ assert_eq!(
+ distinct_count(&mut acc_single),
+ distinct_count(&mut acc_split)
+ );
+ assert_eq!(distinct_count(&mut acc_single), 3);
+ }
+ }
fn h(v: u64) -> u64 {
HLL_HASH_STATE.hash_one(v)
@@ -856,13 +948,6 @@ mod tests {
buf
}
- fn distinct_count(acc: &mut HLLAccumulator) -> u64 {
- match acc.evaluate().unwrap() {
- ScalarValue::UInt64(Some(v)) => v,
- other => panic!("unexpected evaluate result: {other:?}"),
- }
- }
-
#[test]
fn sparse_stays_sparse_for_small_groups() {
let mut g = GroupHll::default();
@@ -980,83 +1065,4 @@ mod tests {
dst.merge_serialized(&bytes).unwrap();
assert_eq!(dst.count(), 0);
}
-
- /// `approx_distinct(v) FILTER (WHERE nullable_bool)` — a NULL filter row
- /// must not be counted (null filter is treated the same as false).
- #[test]
- fn update_batch_nullable_filter_excludes_null_filter_rows() {
- let values: ArrayRef = Arc::new(Int64Array::from(vec![1i64, 2, 3, 4, 5]));
- // row 0: filter=true, row 1: filter=NULL, row 2: filter=false,
- // row 3: filter=NULL, row 4: filter=true
- let filter =
- BooleanArray::from(vec![Some(true), None, Some(false), None, Some(true)]);
-
- let mut acc = HllGroupsAccumulator::new();
- // put all rows in group 0
- let group_indices = vec![0usize; 5];
- acc.update_batch(&[values], &group_indices, Some(&filter), 1)
- .unwrap();
-
- // Only rows 0 and 4 (values 1 and 5) should be counted.
- let result = acc.evaluate(EmitTo::All).unwrap();
- let counts = result.as_any().downcast_ref::<UInt64Array>().unwrap();
- // reference: hash 1 and 5 into a dense sketch
- let expected = reference_count(&[h(1), h(5)]);
- assert_eq!(counts.value(0), expected);
- }
-
- /// Regression: a short (≤ 12-byte) Utf8View string must hash identically
- /// in an all-inline batch and in a mixed batch that also contains a long
- /// string (which forces a data buffer).
- #[test]
- fn utf8view_groups_short_string_hashed_consistently_across_batches() {
- // Batch 1: all-inline (no data buffers) — "aaa" is hashed as u128 view.
- let batch1: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", "bbb"]));
- assert!(batch1.as_string_view().data_buffers().is_empty());
-
- // Batch 2: mixed — LONG forces a data buffer; "aaa" must still be
- // hashed as u128 view so it matches its appearance in batch 1.
- let batch2: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", LONG]));
- assert!(!batch2.as_string_view().data_buffers().is_empty());
-
- let group_indices = vec![0usize, 0];
- let mut acc = HllGroupsAccumulator::new();
- acc.update_batch(&[batch1], &group_indices, None, 1)
- .unwrap();
- acc.update_batch(&[batch2], &group_indices, None, 1)
- .unwrap();
-
- // True distinct values: {"aaa", "bbb", LONG} == 3.
- let result = acc.evaluate(EmitTo::All).unwrap();
- let counts = result.as_any().downcast_ref::<UInt64Array>().unwrap();
- assert_eq!(counts.value(0), 3);
- }
-
- /// Regression: a short (≤ 12-byte) Utf8View string must hash identically
- /// regardless of which batch it appears in — all-inline or mixed.
- #[test]
- fn utf8view_acc_split_batches_match_single_mixed_batch() {
- // Multiset: {"aaa" x2, "bbb", LONG}, so 3 distinct values.
- let mixed: ArrayRef =
- Arc::new(StringViewArray::from(vec!["aaa", "bbb", LONG, "aaa"]));
- let mut acc_single = HLLAccumulator::new();
- acc_single.update_batch(&[mixed]).unwrap();
-
- // Same multiset, but split so "aaa" lands in both an all-inline batch
- // and a batch with a data buffer (forced by LONG).
- let inline_only: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", "bbb"]));
- let with_buffer: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", LONG]));
- assert!(inline_only.as_string_view().data_buffers().is_empty());
- assert!(!with_buffer.as_string_view().data_buffers().is_empty());
-
- let mut acc_split = HLLAccumulator::new();
- acc_split.update_batch(&[inline_only]).unwrap();
- acc_split.update_batch(&[with_buffer]).unwrap();
-
- assert_eq!(
- distinct_count(&mut acc_single),
- distinct_count(&mut acc_split)
- );
- assert_eq!(distinct_count(&mut acc_single), 3);
- }
}