Skip to content

Apache DataFusion SQL Query Engine

fix: approx_distinct over-counts for utf8view (#22815)

Huaijin84dcc0b30e4e
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index 3550035635..38b902964f 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -251,16 +251,29 @@ impl Accumulator for StringViewHLLAccumulator {
fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
let array: &StringViewArray = downcast_value!(values[0], StringViewArray);
- // When all strings are stored inline in the StringView (≤ 12 bytes),
- // hash the raw u128 view directly instead of materializing a &str.
if array.data_buffers().is_empty() {
+ // Fast path: with no data buffers every value is inline, so they all
+ // take the u128 path — no need to check the length per row.
for (i, &view) in array.views().iter().enumerate() {
if !array.is_null(i) {
self.hll.add_hashed(HLL_HASH_STATE.hash_one(view));
}
}
} else {
- self.hll.extend(array.iter().flatten());
+ // Mixed batch: decide per row by length. Short strings still use the
+ // u128 path so they match how they'd be hashed in an all-inline
+ // batch; only the genuinely out-of-line strings materialize a &str.
+ for (i, &view) in array.views().iter().enumerate() {
+ if array.is_null(i) {
+ continue;
+ }
+ // The low 32 bits of the u128 view encode the string length.
+ if (view as u32) <= 12 {
+ self.hll.add_hashed(HLL_HASH_STATE.hash_one(view));
+ } else {
+ self.hll.add(array.value(i));
+ }
+ }
}
Ok(())
@@ -567,9 +580,17 @@ impl HllValueHasher for Utf8ViewHasher {
}
}
} else {
+ // Mixed batch: short strings (≤ 12 bytes) are still inline and must
+ // be hashed as the raw u128 view to match the all-inline fast path.
+ let views = array.views();
for i in 0..array.len() {
if nulls.is_none_or(|n| n.is_valid(i)) {
- f(i, HLL_HASH_STATE.hash_one(array.value(i)));
+ let view = views[i];
+ if (view as u32) <= 12 {
+ f(i, HLL_HASH_STATE.hash_one(view));
+ } else {
+ f(i, HLL_HASH_STATE.hash_one(array.value(i)));
+ }
}
}
}
@@ -1037,10 +1058,14 @@ fn is_hll_groups_type(data_type: &DataType) -> bool {
}
#[cfg(test)]
-mod groups_tests {
+mod tests {
use super::*;
+ use arrow::array::{AsArray, Int64Array, StringViewArray};
+ use std::sync::Arc;
+
+ // A string longer than the 12-byte inline limit
+ const LONG: &str = "this string is definitely longer than twelve bytes";
- /// Hash a value the same way the accumulators do.
fn h(v: u64) -> u64 {
HLL_HASH_STATE.hash_one(v)
}
@@ -1061,6 +1086,13 @@ mod groups_tests {
buf
}
+ fn distinct_count(acc: &mut StringViewHLLAccumulator) -> u64 {
+ match acc.evaluate().unwrap() {
+ ScalarValue::UInt64(Some(v)) => v,
+ other => panic!("unexpected evaluate result: {other:?}"),
+ }
+ }
+
#[test]
fn sparse_stays_sparse_for_small_groups() {
let mut g = GroupHll::default();
@@ -1183,9 +1215,6 @@ mod groups_tests {
/// must not be counted (null filter is treated the same as false).
#[test]
fn update_batch_nullable_filter_excludes_null_filter_rows() {
- use arrow::array::Int64Array;
- use std::sync::Arc;
-
let values: ArrayRef = Arc::new(Int64Array::from(vec![1i64, 2, 3, 4, 5]));
// row 0: filter=true, row 1: filter=NULL, row 2: filter=false,
// row 3: filter=NULL, row 4: filter=true
@@ -1205,4 +1234,59 @@ mod groups_tests {
let expected = reference_count(&[h(1), h(5)]);
assert_eq!(counts.value(0), expected);
}
+
+ /// Regression: a short (≤ 12-byte) Utf8View string must hash identically
+ /// in an all-inline batch and in a mixed batch that also contains a long
+ /// string (which forces a data buffer).
+ #[test]
+ fn utf8view_groups_short_string_hashed_consistently_across_batches() {
+ // Batch 1: all-inline (no data buffers) — "aaa" is hashed as u128 view.
+ let batch1: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", "bbb"]));
+ assert!(batch1.as_string_view().data_buffers().is_empty());
+
+ // Batch 2: mixed — LONG forces a data buffer; "aaa" must still be
+ // hashed as u128 view so it matches its appearance in batch 1.
+ let batch2: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", LONG]));
+ assert!(!batch2.as_string_view().data_buffers().is_empty());
+
+ let group_indices = vec![0usize, 0];
+ let mut acc = HllGroupsAccumulator::<Utf8ViewHasher>::new();
+ acc.update_batch(&[batch1], &group_indices, None, 1)
+ .unwrap();
+ acc.update_batch(&[batch2], &group_indices, None, 1)
+ .unwrap();
+
+ // True distinct values: {"aaa", "bbb", LONG} == 3.
+ let result = acc.evaluate(EmitTo::All).unwrap();
+ let counts = result.as_any().downcast_ref::<UInt64Array>().unwrap();
+ assert_eq!(counts.value(0), 3);
+ }
+
+ /// Regression: a short (≤ 12-byte) Utf8View string must hash identically
+ /// regardless of which batch it appears in — all-inline or mixed.
+ #[test]
+ fn utf8view_acc_split_batches_match_single_mixed_batch() {
+ // Multiset: {"aaa" x2, "bbb", LONG}, so 3 distinct values.
+ let mixed: ArrayRef =
+ Arc::new(StringViewArray::from(vec!["aaa", "bbb", LONG, "aaa"]));
+ let mut acc_single = StringViewHLLAccumulator::new();
+ acc_single.update_batch(&[mixed]).unwrap();
+
+ // Same multiset, but split so "aaa" lands in both an all-inline batch
+ // and a batch with a data buffer (forced by LONG).
+ let inline_only: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", "bbb"]));
+ let with_buffer: ArrayRef = Arc::new(StringViewArray::from(vec!["aaa", LONG]));
+ assert!(inline_only.as_string_view().data_buffers().is_empty());
+ assert!(!with_buffer.as_string_view().data_buffers().is_empty());
+
+ let mut acc_split = StringViewHLLAccumulator::new();
+ acc_split.update_batch(&[inline_only]).unwrap();
+ acc_split.update_batch(&[with_buffer]).unwrap();
+
+ assert_eq!(
+ distinct_count(&mut acc_single),
+ distinct_count(&mut acc_split)
+ );
+ assert_eq!(distinct_count(&mut acc_single), 3);
+ }
}