Apache DataFusion SQL Query Engine
perf: Concretely typed TopK StringHeap, and StringHashTable slot reuse (#24105)
Michal Piatkowski6663e4348049
diff --git a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs b/datafusion/physical-plan/src/aggregates/topk/hash_table.rsindex 61410893df..117cf9b378 100644--- a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs+++ b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs@@ -30,28 +30,27 @@ use datafusion_common::Result;use datafusion_common::exec_datafusion_err;use datafusion_common::hash_utils::RandomState;use half::f16;-use hashbrown::hash_table::HashTable;+use hashbrown::hash_table::{Entry, HashTable};+use std::borrow::BorrowMut;use std::fmt::Debug;use std::hash::BuildHasher;use std::sync::Arc;-/// A "type alias" for Keys which are stored in our map-pub trait KeyType: Clone + Comparable + Debug {}+/// `heap_idx` assigned to groups whose aggregate values are all NULL.+/// Such groups are tracked in the hash table only, without entering heap,+/// so they can be emitted with a NULL aggregate value at the end.+const NULL_HEAP_IDX: usize = usize::MAX - 1;-impl<T> KeyType for T where T: Clone + Comparable + Debug {}--/// `heap_idx` assigned to groups whose aggregate values are all NULL. Such-/// groups are tracked in the hash table only (they never enter the heap), so-/// they can be emitted with a NULL aggregate value at the end.-const NULL_HEAP_IDX: usize = usize::MAX;+/// `heap_idx` needs to distinguish vacant slots from items that have a NULL key and value.+const VACANT_HEAP_IDX: usize = usize::MAX;/// An entry in our hash table that:/// 1. memoizes the hash/// 2. contains the key (ID)/// 3. contains the value (heap_idx - an index into the corresponding heap)-pub struct HashTableItem<ID: KeyType> {+pub struct HashTableItem<ID> {hash: u64,- pub id: ID,+ pub id: Option<ID>,pub heap_idx: usize,}@@ -59,12 +58,14 @@ pub struct HashTableItem<ID: KeyType> {/// 1. limits the number of entries to the top K/// 2. Allocates a capacity greater than top K to maintain a low-fill factor and prevent resizing/// 3. Tracks indexes to allow corresponding heap to refer to entries by index vs hash-struct TopKHashTable<ID: KeyType> {+struct TopKHashTable<ID> {map: HashTable<usize>,// Store the actual items separately to allow for index-based access- store: Vec<Option<HashTableItem<ID>>>,+ store: Vec<HashTableItem<ID>>,// Free indexes in the store for reusefree_indices: Vec<usize>,+ // Pool of reusable value locations, usually Strings+ free_slots: Vec<ID>,// The maximum number of entries allowedlimit: usize,// Number of entries registered as all-NULL (heap_idx == NULL_HEAP_IDX)@@ -103,8 +104,8 @@ pub trait ArrowHashTable {/// Remove the group at `row_idx` if it is registered as all-NULL. Returns/// true if a NULL registration was removed.fn remove_if_null(&mut self, row_idx: usize) -> bool;- /// Store indexes of all groups registered as all-NULL- fn null_map_idxs(&self) -> Vec<usize>;+ /// Store indexes of all groups registered as all-NULL. Returns null count.+ fn null_map_idxs(&self, out: &mut Vec<usize>) -> usize;}/// Returns true if the given data type can be used as a top-K aggregation hash key.@@ -126,7 +127,7 @@ wherefor<'a> &'a S: StringArrayType<'a>,{owned: S,- map: TopKHashTable<Option<String>>,+ map: TopKHashTable<String>,rnd: RandomState,}@@ -136,8 +137,9 @@ whereOption<<VAL as ArrowPrimitiveType>::Native>: Comparable,{owned: PrimitiveArray<VAL>,- map: TopKHashTable<Option<VAL::Native>>,+ map: TopKHashTable<VAL::Native>,rnd: RandomState,+ value_type: DataType,}impl<S> StringHashTable<S>@@ -146,9 +148,8 @@ wherefor<'a> &'a S: StringArrayType<'a>,{pub fn new(limit: usize) -> Self {- let owned = S::from(Vec::new());Self {- owned,+ owned: S::from(Vec::new()),map: TopKHashTable::new(limit, limit * 10),rnd: RandomState::default(),}@@ -200,19 +201,14 @@ wherelet hash = self.rnd.hash_one(id);// Use entry API to avoid double lookup- self.map.find_or_insert(- hash,- id.map(ToOwned::to_owned),- replace_idx,- Self::eq_fn(id),- )+ self.map+ .find_or_insert(hash, id, replace_idx, Self::eq_fn(id))}fn insert_null(&mut self, row_idx: usize) -> bool {let id = some_value(&self.owned, row_idx);let hash = self.rnd.hash_one(id);- self.map- .insert_null(hash, id.map(ToOwned::to_owned), Self::eq_fn(id))+ self.map.insert_null(hash, id, Self::eq_fn(id))}fn remove_if_null(&mut self, row_idx: usize) -> bool {@@ -221,8 +217,8 @@ whereself.map.remove_if_null(hash, Self::eq_fn(id))}- fn null_map_idxs(&self) -> Vec<usize> {- self.map.null_map_idxs()+ fn null_map_idxs(&self, out: &mut Vec<usize>) -> usize {+ self.map.null_map_idxs(out)}}@@ -230,14 +226,12 @@ impl<VAL: ArrowPrimitiveType> PrimitiveHashTable<VAL>whereOption<<VAL as ArrowPrimitiveType>::Native>: Comparable + HashValue,{- pub fn new(limit: usize, kt: DataType) -> Self {- let owned = PrimitiveArray::<VAL>::builder(0)- .with_data_type(kt)- .finish();+ pub fn new(limit: usize, value_type: DataType) -> Self {Self {- owned,+ owned: PrimitiveArray::<VAL>::new_null(0).with_data_type(value_type.clone()),map: TopKHashTable::new(limit, limit * 10),rnd: RandomState::default(),+ value_type,}}@@ -277,13 +271,10 @@ wherefn take_all(&mut self, indexes: Vec<usize>) -> ArrayRef {let ids = self.map.take_all(indexes);- let mut builder: PrimitiveBuilder<VAL> = PrimitiveArray::builder(ids.len())- .with_data_type(self.owned.data_type().clone());+ let mut builder: PrimitiveBuilder<VAL> =+ PrimitiveArray::builder(ids.len()).with_data_type(self.value_type.clone());for id in ids.into_iter() {- match id {- None => builder.append_null(),- Some(id) => builder.append_value(id),- }+ builder.append_option(id);}let ids = builder.finish();Arc::new(ids)@@ -297,12 +288,12 @@ wherelet (id, hash) = self.id_and_hash(row_idx);// Use entry API to avoid double lookupself.map- .find_or_insert(hash, id, replace_idx, Self::eq_fn(id))+ .find_or_insert(hash, id.as_ref(), replace_idx, Self::eq_fn(id))}fn insert_null(&mut self, row_idx: usize) -> bool {let (id, hash) = self.id_and_hash(row_idx);- self.map.insert_null(hash, id, Self::eq_fn(id))+ self.map.insert_null(hash, id.as_ref(), Self::eq_fn(id))}fn remove_if_null(&mut self, row_idx: usize) -> bool {@@ -310,39 +301,47 @@ whereself.map.remove_if_null(hash, Self::eq_fn(id))}- fn null_map_idxs(&self) -> Vec<usize> {- self.map.null_map_idxs()+ fn null_map_idxs(&self, out: &mut Vec<usize>) -> usize {+ self.map.null_map_idxs(out)}}-use hashbrown::hash_table::Entry;-impl<ID: KeyType + PartialEq> TopKHashTable<ID> {+impl<ID: PartialEq> TopKHashTable<ID> {pub fn new(limit: usize, capacity: usize) -> Self {Self {map: HashTable::with_capacity(capacity),store: Vec::with_capacity(capacity),free_indices: Vec::new(),+ free_slots: Vec::new(),limit,null_count: 0,}}pub fn heap_idx_at(&self, map_idx: usize) -> usize {- self.store[map_idx].as_ref().unwrap().heap_idx+ self.store[map_idx].heap_idx}/// Remove the entry stored at `map_idx`, freeing its store slot for reusefn remove_at(&mut self, map_idx: usize) {- let item_to_remove = self.store[map_idx].as_ref().unwrap();+ let item_to_remove = &self.store[map_idx];let hash = item_to_remove.hash;let id_to_remove = &item_to_remove.id;- let eq = |&idx: &usize| self.store[idx].as_ref().unwrap().id == *id_to_remove;- let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;+ let eq = |idx: &usize| self.store[*idx].id == *id_to_remove;+ let hasher = |idx: &usize| self.store[*idx].hash;match self.map.entry(hash, eq, hasher) {Entry::Occupied(entry) => {let (removed_idx, _) = entry.remove();- self.store[removed_idx] = None;+ let mi = &mut self.store[removed_idx];+ debug_assert!(mi.is_occupied());+ mi.heap_idx = VACANT_HEAP_IDX;+ match mi.id.take() {+ Some(slot) if Self::use_free_slots() => {+ self.free_slots.push(slot);+ }+ _ => (),+ }self.free_indices.push(removed_idx);}Entry::Vacant(_) => unreachable!(),@@ -363,63 +362,91 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {fn update_heap_idx(&mut self, mapper: &[(usize, usize)]) {for (m, h) in mapper {- self.store[*m].as_mut().unwrap().heap_idx = *h;+ self.store[*m].heap_idx = *h;}}+ /// Used to avoid pushing pointless copies of primitives to the `free_slots` pool.+ const fn use_free_slots() -> bool {+ std::mem::needs_drop::<ID>()+ }+/// Find an existing entry or insert a new one, avoiding double hash table lookup./// Returns (map_idx, kind) where kind describes whether the group already/// existed, was newly inserted, or was converted from an all-NULL group./// If inserting a new entry and the table is full, replaces the entry at replace_idx.- pub fn find_or_insert(+ pub fn find_or_insert<Q>(&mut self,hash: u64,- id: ID,+ id: Option<&Q>,replace_idx: usize,- mut eq: impl FnMut(&ID) -> bool,- ) -> (usize, InsertKind) {+ mut eq: impl FnMut(&Option<ID>) -> bool,+ ) -> (usize, InsertKind)+ where+ Q: ToOwned<Owned = ID> + ?Sized,+ {// Check if entry exists - this is the only hash table lookuplet mut replaced_null = false;- {- let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);- if let Some(&map_idx) = self.map.find(hash, eq_fn) {- if self.store[map_idx].as_ref().unwrap().heap_idx == NULL_HEAP_IDX {- // This group was registered as all-NULL but now produced a- // value: unregister it so it is inserted as a valued group- self.remove_at(map_idx);- self.null_count -= 1;- replaced_null = true;- } else {- return (map_idx, InsertKind::Existing);- }++ let eq_fn = |idx: &usize| eq(&self.store[*idx].id);+ if let Some(&map_idx) = self.map.find(hash, eq_fn) {+ if self.store[map_idx].is_null() {+ // This group was registered as all-NULL but now produced a+ // value: unregister it so it is inserted as a valued group+ self.remove_at(map_idx);+ self.null_count -= 1;+ replaced_null = true;+ } else {+ return (map_idx, InsertKind::Existing);}}// Entry doesn't exist - compute heap_idx and prepare itemlet heap_idx = self.remove_if_full(replace_idx);+ let store_idx = self.push_store_item(hash, id, heap_idx);+ let kind = if replaced_null {+ InsertKind::ReplacedNull+ } else {+ InsertKind::New+ };+ (store_idx, kind)+ }++ fn push_store_item<Q>(&mut self, hash: u64, id: Option<&Q>, heap_idx: usize) -> usize+ where+ Q: ToOwned<Owned = ID> + ?Sized,+ {+ let id = if Self::use_free_slots() {+ id.map(|id| match self.free_slots.pop() {+ Some(mut slot) => {+ id.clone_into(&mut slot);+ slot+ }+ _ => id.to_owned(),+ })+ } else {+ debug_assert!(self.free_slots.is_empty(), "primitives should not pool");+ id.map(ToOwned::to_owned)+ };let mi = HashTableItem::new(hash, id, heap_idx);let store_idx = if let Some(idx) = self.free_indices.pop() {- self.store[idx] = Some(mi);+ debug_assert!(self.store[idx].is_vacant(), "slot should be empty");+ self.store[idx] = mi;idx} else {- self.store.push(Some(mi));+ self.store.push(mi);self.store.len() - 1};// Reserve space if needed- let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;+ let hasher = |idx: &usize| self.store[*idx].hash;if self.map.len() == self.map.capacity() {self.map.reserve(self.limit, hasher);}// Insert without checking again since we already confirmed it doesn't existself.map.insert_unique(hash, store_idx, hasher);- let kind = if replaced_null {- InsertKind::ReplacedNull- } else {- InsertKind::New- };- (store_idx, kind)+ store_idx}/// Register a group whose aggregate values are all NULL, unless it is@@ -427,13 +454,17 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {/// never enter the heap. At most `limit` NULL groups are tracked: they all/// tie on the sort key, so any `limit` of them is a valid top-k superset./// Returns true if the group was newly registered.- pub fn insert_null(+ pub fn insert_null<Q>(&mut self,hash: u64,- id: ID,- mut eq: impl FnMut(&ID) -> bool,- ) -> bool {- let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);+ id: Option<&Q>,+ mut eq: impl FnMut(&Option<ID>) -> bool,+ ) -> bool+ where+ Q: ToOwned<Owned = ID> + ?Sized,+ ID: BorrowMut<Q::Owned>,+ {+ let eq_fn = |idx: &usize| eq(&self.store[*idx].id);if self.map.find(hash, eq_fn).is_some() {return false;}@@ -442,20 +473,7 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {return false;}- let mi = HashTableItem::new(hash, id, NULL_HEAP_IDX);- let store_idx = if let Some(idx) = self.free_indices.pop() {- self.store[idx] = Some(mi);- idx- } else {- self.store.push(Some(mi));- self.store.len() - 1- };-- let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;- if self.map.len() == self.map.capacity() {- self.map.reserve(self.limit, hasher);- }- self.map.insert_unique(hash, store_idx, hasher);+ _ = self.push_store_item(hash, id, NULL_HEAP_IDX);self.null_count += 1;true}@@ -464,10 +482,14 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {/// all-NULL group produces a value that loses to the current top-k: the/// group can no longer reach the top-k, but it must not be emitted with a/// NULL value either. Returns true if a NULL registration was removed.- pub fn remove_if_null(&mut self, hash: u64, mut eq: impl FnMut(&ID) -> bool) -> bool {- let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);+ pub fn remove_if_null(+ &mut self,+ hash: u64,+ mut eq: impl FnMut(&Option<ID>) -> bool,+ ) -> bool {+ let eq_fn = |idx: &usize| eq(&self.store[*idx].id);if let Some(&map_idx) = self.map.find(hash, eq_fn)- && self.store[map_idx].as_ref().unwrap().heap_idx == NULL_HEAP_IDX+ && self.store[map_idx].is_null(){self.remove_at(map_idx);self.null_count -= 1;@@ -477,39 +499,57 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {}/// Store indexes of all groups registered as all-NULL- pub fn null_map_idxs(&self) -> Vec<usize> {- self.store+ pub fn null_map_idxs(&self, out: &mut Vec<usize>) -> usize {+ let iter = self+ .store.iter().enumerate()- .filter_map(|(idx, item)| {- item.as_ref()- .filter(|item| item.heap_idx == NULL_HEAP_IDX)- .map(|_| idx)- })- .collect()+ .filter_map(|(idx, item)| item.is_null().then_some(idx));++ let prev_len = out.len();+ out.extend(iter);++ debug_assert_eq!(out.len(), self.null_count + prev_len);+ self.null_count}pub fn len(&self) -> usize {self.map.len()}- pub fn take_all(&mut self, idxs: Vec<usize>) -> Vec<ID> {+ pub fn take_all(&mut self, idxs: Vec<usize>) -> Vec<Option<ID>> {let ids = idxs.into_iter()- .map(|idx| self.store[idx].take().unwrap().id)+ .map(|idx| self.store[idx].id.take()).collect();self.map.clear();self.store.clear();self.free_indices.clear();+ // No need to clear free_slots.self.null_count = 0;ids}}-impl<ID: KeyType> HashTableItem<ID> {- pub fn new(hash: u64, id: ID, heap_idx: usize) -> Self {+impl<ID> HashTableItem<ID> {+ pub fn new(hash: u64, id: Option<ID>, heap_idx: usize) -> Self {Self { hash, id, heap_idx }}++ #[inline]+ pub fn is_null(&self) -> bool {+ self.heap_idx == NULL_HEAP_IDX+ }++ #[inline]+ pub fn is_vacant(&self) -> bool {+ self.heap_idx == VACANT_HEAP_IDX+ }++ #[inline]+ pub fn is_occupied(&self) -> bool {+ self.heap_idx != VACANT_HEAP_IDX+ }}impl HashValue for Option<String> {@@ -585,6 +625,13 @@ mod tests {use arrow_schema::TimeUnit;use std::collections::BTreeMap;+ fn assert_nulls<ID: PartialEq>(map: &TopKHashTable<ID>, expected: &[usize]) {+ assert_eq!(map.null_count, expected.len());+ let mut actual = Vec::new();+ assert_eq!(map.null_map_idxs(&mut actual), expected.len());+ assert_eq!(actual, expected);+ }+#[test]fn should_emit_correct_type() -> Result<()> {let ids =@@ -603,14 +650,14 @@ mod tests {fn should_resize_properly() -> Result<()> {let mut heap_to_map = BTreeMap::<usize, usize>::new();// Create TopKHashTable with limit=5 and capacity=3 to force resizing- let mut map = TopKHashTable::<Option<String>>::new(5, 3);+ let mut map = TopKHashTable::<String>::new(5, 3);// Insert 5 entries, tracking the heap-to-map index mappingfor (heap_idx, id) in ["1", "2", "3", "4", "5"].iter().enumerate() {let value = Some(id.to_string());let hash = heap_idx as u64;let (map_idx, kind) =- map.find_or_insert(hash, value.clone(), heap_idx, |v| *v == value);+ map.find_or_insert(hash, value.as_ref(), heap_idx, |v| *v == value);assert_eq!(kind, InsertKind::New, "Entry should be new");heap_to_map.insert(heap_idx, map_idx);}@@ -636,32 +683,29 @@ mod tests {#[test]fn should_track_null_groups() -> Result<()> {- let mut map = TopKHashTable::<Option<String>>::new(2, 10);+ let mut map = TopKHashTable::<String>::new(2, 10);let a = Some("a".to_string());let b = Some("b".to_string());let c = Some("c".to_string());// register two all-NULL groups; the third exceeds the NULL group limit- assert!(map.insert_null(100, a.clone(), |v| *v == a));- assert!(map.insert_null(200, b.clone(), |v| *v == b));- assert!(!map.insert_null(300, c.clone(), |v| *v == c));+ assert!(map.insert_null(100, a.as_ref(), |v| *v == a));+ assert!(map.insert_null(200, b.as_ref(), |v| *v == b));+ assert!(!map.insert_null(300, c.as_ref(), |v| *v == c));// re-registering an existing NULL group is a no-op- assert!(!map.insert_null(100, a.clone(), |v| *v == a));- assert_eq!(map.null_count, 2);- assert_eq!(map.null_map_idxs(), vec![0, 1]);+ assert!(!map.insert_null(100, a.as_ref(), |v| *v == a));+ assert_nulls(&map, &[0, 1]);// a valued insert for a NULL group converts it to a valued group- let (map_idx, kind) = map.find_or_insert(200, b.clone(), 0, |v| *v == b);+ let (map_idx, kind) = map.find_or_insert(200, b.as_ref(), 0, |v| *v == b);assert_eq!(kind, InsertKind::ReplacedNull, "NULL group should convert");assert_eq!(map.heap_idx_at(map_idx), 0, "Heap should append at 0");- assert_eq!(map.null_count, 1);- assert_eq!(map.null_map_idxs(), vec![0]);+ assert_nulls(&map, &[0]);// remove the remaining NULL group; removing twice is a no-opmap.remove_if_null(100, |v| *v == a);- assert_eq!(map.null_count, 0);- assert!(map.null_map_idxs().is_empty());+ assert_nulls(&map, &[]);map.remove_if_null(100, |v| *v == a);// removing a valued group via remove_if_null is a no-opmap.remove_if_null(200, |v| *v == b);@@ -670,26 +714,43 @@ mod tests {Ok(())}+ #[test]+ fn should_track_null_id() {+ let mut map = TopKHashTable::<String>::new(2, 10);++ let hash = 0;+ let none = Option::<&str>::None;++ assert!(map.insert_null(hash, none, Option::is_none));+ assert_nulls(&map, &[0]);++ assert!(!map.insert_null(hash, none, Option::is_none));+ assert_nulls(&map, &[0]);++ assert!(map.remove_if_null(hash, Option::is_none));+ assert_nulls(&map, &[]);+ }+#[test]fn should_reuse_all_freed_store_slots() -> Result<()> {- let mut map = TopKHashTable::<Option<String>>::new(1, 10);+ let mut map = TopKHashTable::<String>::new(1, 10);let a = Some("a".to_string());let b = Some("b".to_string());let c = Some("c".to_string());- let (b_idx, kind) = map.find_or_insert(100, b.clone(), 0, |v| *v == b);+ let (b_idx, kind) = map.find_or_insert(100, b.as_ref(), 0, |v| *v == b);assert_eq!(kind, InsertKind::New);- assert!(map.insert_null(200, a.clone(), |v| *v == a));+ assert!(map.insert_null(200, a.as_ref(), |v| *v == a));// Converting a NULL group while the valued heap is full frees two// slots: the NULL registration and the evicted valued group.- let (_, kind) = map.find_or_insert(200, a.clone(), b_idx, |v| *v == a);+ let (_, kind) = map.find_or_insert(200, a.as_ref(), b_idx, |v| *v == a);assert_eq!(kind, InsertKind::ReplacedNull);// Both freed slots must remain reusable. Otherwise repeated// conversions make the backing store grow without bound.- assert!(map.insert_null(300, c.clone(), |v| *v == c));+ assert!(map.insert_null(300, c.as_ref(), |v| *v == c));assert_eq!(map.store.len(), 2);Ok(())diff --git a/datafusion/physical-plan/src/aggregates/topk/heap.rs b/datafusion/physical-plan/src/aggregates/topk/heap.rsindex 7982d94d39..89ddaaa0be 100644--- a/datafusion/physical-plan/src/aggregates/topk/heap.rs+++ b/datafusion/physical-plan/src/aggregates/topk/heap.rs@@ -23,12 +23,10 @@//! Supported value types include Arrow primitives (integers, floats, decimals, intervals)//! and UTF-8 strings (`Utf8`, `LargeUtf8`, `Utf8View`) using lexicographic ordering.-use arrow::array::{ArrayRef, ArrowPrimitiveType, PrimitiveArray, downcast_primitive};-use arrow::array::{LargeStringBuilder, StringBuilder, StringViewBuilder};+use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano};use arrow::array::{- StringArray,- cast::AsArray,- types::{IntervalDayTime, IntervalMonthDayNano},+ Array, ArrayAccessor, ArrayRef, ArrowPrimitiveType, AsArray, LargeStringArray,+ PrimitiveArray, StringArray, StringArrayType, StringViewArray, downcast_primitive,};use arrow::buffer::ScalarBuffer;use arrow::datatypes::{DataType, i256};@@ -42,8 +40,20 @@ use std::fmt::{Debug, Display, Formatter};use std::sync::Arc;/// A custom version of `Ord` that only exists to we can implement it for the Values in our heap-pub trait Comparable {- fn comp(&self, other: &Self) -> Ordering;+pub trait Comparable<Rhs: ?Sized = Self> {+ fn comp(&self, other: &Rhs) -> Ordering;+}++impl Comparable<String> for str {+ fn comp(&self, other: &String) -> Ordering {+ self.cmp(other.as_str())+ }+}++impl Comparable for String {+ fn comp(&self, other: &Self) -> Ordering {+ self.cmp(other)+ }}impl Comparable for Option<String> {@@ -53,15 +63,52 @@ impl Comparable for Option<String> {}/// A "type alias" for Values which are stored in our heap-pub trait ValueType: Comparable + Clone + Debug {}+pub trait ValueType: Comparable + Clone + Debug + Default {}++impl<T> ValueType for T where T: Comparable + Clone + Debug + Default {}-impl<T> ValueType for T where T: Comparable + Clone + Debug {}+const VACANT_MAP_IDX: usize = usize::MAX;/// An entry in our heap, which contains both the value and a index into an external HashTablestruct HeapItem<VAL: ValueType> {val: VAL,map_idx: usize,}+impl<VAL: ValueType> HeapItem<VAL> {+ fn is_vacant(&self) -> bool {+ self.map_idx == VACANT_MAP_IDX+ }++ fn is_occupied(&self) -> bool {+ self.map_idx != VACANT_MAP_IDX+ }++ fn map_idx(&self) -> Option<usize> {+ self.is_occupied().then_some(self.map_idx)+ }++ fn take(&mut self) -> Option<Self> {+ self.is_occupied().then(|| std::mem::take(self))+ }++ fn as_ref(&self) -> &Self {+ debug_assert!(self.is_occupied());+ self+ }++ fn as_mut(&mut self) -> &mut Self {+ debug_assert!(self.is_occupied());+ self+ }+}+impl<VAL: ValueType> Default for HeapItem<VAL> {+ fn default() -> Self {+ Self {+ val: VAL::default(),+ map_idx: VACANT_MAP_IDX,+ }+ }+}/// A custom heap implementation that allows several things that couldn't be achieved with/// `collections::BinaryHeap`:@@ -73,11 +120,12 @@ struct TopKHeap<VAL: ValueType> {desc: bool,len: usize,capacity: usize,- heap: Vec<Option<HeapItem<VAL>>>,+ heap: Vec<HeapItem<VAL>>,}/// An interface to hide the generic type signature of TopKHeap behind arrow arrayspub trait ArrowHeap {+ fn value_type(&self) -> &DataType;fn set_batch(&mut self, vals: ArrayRef);fn is_worse(&self, idx: usize) -> bool;fn worst_map_idx(&self) -> usize;@@ -99,20 +147,19 @@ wherebatch: PrimitiveArray<VAL>,heap: TopKHeap<VAL::Native>,desc: bool,- data_type: DataType,+ value_type: DataType,}impl<VAL: ArrowPrimitiveType> PrimitiveHeap<VAL>where<VAL as ArrowPrimitiveType>::Native: Comparable,{- pub fn new(limit: usize, desc: bool, data_type: DataType) -> Self {- let batch = PrimitiveArray::<VAL>::builder(0).finish();+ pub fn new(limit: usize, desc: bool, value_type: DataType) -> Self {Self {- batch,+ batch: PrimitiveArray::<VAL>::new_null(0).with_data_type(value_type.clone()),heap: TopKHeap::new(limit, desc),desc,- data_type,+ value_type,}}}@@ -121,8 +168,12 @@ impl<VAL: ArrowPrimitiveType> ArrowHeap for PrimitiveHeap<VAL>where<VAL as ArrowPrimitiveType>::Native: Comparable,{+ fn value_type(&self) -> &DataType {+ &self.value_type+ }+fn set_batch(&mut self, vals: ArrayRef) {- self.batch = PrimitiveArray::from(vals.to_data());+ self.batch = vals.as_primitive().clone();}fn is_worse(&self, row_idx: usize) -> bool {@@ -140,7 +191,7 @@ wherefn insert(&mut self, row_idx: usize, map_idx: usize, map: &mut Vec<(usize, usize)>) {let new_val = self.batch.value(row_idx);- self.heap.append_or_replace(new_val, map_idx, map);+ self.heap.append_or_replace(&new_val, map_idx, map);}fn replace_if_better(@@ -150,14 +201,14 @@ wheremap: &mut Vec<(usize, usize)>,) {let new_val = self.batch.value(row_idx);- self.heap.replace_if_better(heap_idx, new_val, map);+ self.heap.replace_if_better(&new_val, heap_idx, map);}fn drain(&mut self) -> (ArrayRef, Vec<usize>) {let nulls = None;let (vals, map_idxs) = self.heap.drain();let arr = PrimitiveArray::<VAL>::new(ScalarBuffer::from(vals), nulls)- .with_data_type(self.data_type.clone());+ .with_data_type(self.value_type.clone());(Arc::new(arr), map_idxs)}}@@ -169,58 +220,46 @@ where/// borrowed strings are compared before allocation, and only allocated when the/// heap confirms they improve the top-K set.///-pub struct StringHeap {- batch: ArrayRef,- heap: TopKHeap<Option<String>>,+pub struct StringHeap<S: Array>+where+ for<'a> &'a S: StringArrayType<'a>,+{+ batch: S,+ heap: TopKHeap<String>,desc: bool,- data_type: DataType,}-impl StringHeap {- pub fn new(limit: usize, desc: bool, data_type: DataType) -> Self {- let batch: ArrayRef = Arc::new(StringArray::from(Vec::<&str>::new()));+impl<S> StringHeap<S>+where+ S: Array + From<Vec<Option<String>>>,+ for<'a> &'a S: StringArrayType<'a>,+{+ pub fn new(limit: usize, desc: bool) -> Self {+ let batch = S::from(Vec::new());Self {batch,heap: TopKHeap::new(limit, desc),desc,- data_type,}}-- /// Extracts a string value from the current batch at the given row index.- ///- /// Panics if the row index is out of bounds or if the data type is not one of- /// the supported UTF-8 string types.- ///- /// Note: Null values should not appear in the input; the aggregation layer- /// ensures nulls are filtered before reaching this code.- fn value(&self, row_idx: usize) -> &str {- extract_string_value(&self.batch, &self.data_type, row_idx)- }}-/// Helper to extract a string value from an ArrayRef at a given index.-///-/// Supports `Utf8`, `LargeUtf8`, and `Utf8View` data types.-///-/// # Panics-/// Panics if the index is out of bounds or if the data type is unsupported.-fn extract_string_value<'a>(- batch: &'a ArrayRef,- data_type: &DataType,- idx: usize,-) -> &'a str {- match data_type {- DataType::Utf8 => batch.as_string::<i32>().value(idx),- DataType::LargeUtf8 => batch.as_string::<i64>().value(idx),- DataType::Utf8View => batch.as_string_view().value(idx),- _ => unreachable!("Unsupported string type: {data_type}"),+impl<S> ArrowHeap for StringHeap<S>+where+ S: Array + Clone + From<Vec<String>> + 'static,+ for<'a> &'a S: StringArrayType<'a>,+{+ fn value_type(&self) -> &DataType {+ // Strings don't store any metadata.+ self.batch.data_type()}-}-impl ArrowHeap for StringHeap {fn set_batch(&mut self, vals: ArrayRef) {- self.batch = vals;+ self.batch = vals+ .as_any()+ .downcast_ref::<S>()+ .expect("Unsupported data type")+ .clone();}fn is_worse(&self, row_idx: usize) -> bool {@@ -230,15 +269,9 @@ impl ArrowHeap for StringHeap {// Compare borrowed `&str` against the worst heap value first to avoid// allocating a `String` unless this row would actually replace an// existing heap entry.- let new_val = self.value(row_idx);- let worst_val = self.heap.worst_val().expect("Missing root");- match worst_val {- None => false,- Some(worst_str) => {- (!self.desc && new_val > worst_str.as_str())- || (self.desc && new_val < worst_str.as_str())- }- }+ let new_val = (&self.batch).value(row_idx);+ let worst_val = self.heap.worst_val().expect("Missing root").as_str();+ (!self.desc && new_val > worst_val) || (self.desc && new_val < worst_val)}fn worst_map_idx(&self) -> usize {@@ -246,12 +279,7 @@ impl ArrowHeap for StringHeap {}fn insert(&mut self, row_idx: usize, map_idx: usize, map: &mut Vec<(usize, usize)>) {- // When appending (heap not full) we must allocate to own the string- // because it will be stored in the heap. For replacements we avoid- // allocation until `replace_if_better` confirms a replacement is- // necessary.- let new_str = self.value(row_idx).to_string();- let new_val = Some(new_str);+ let new_val = (&self.batch).value(row_idx);self.heap.append_or_replace(new_val, map_idx, map);}@@ -261,62 +289,14 @@ impl ArrowHeap for StringHeap {row_idx: usize,map: &mut Vec<(usize, usize)>,) {- let new_str = self.value(row_idx);- let existing = self.heap.heap[heap_idx]- .as_ref()- .expect("Missing heap item");-- // Compare borrowed reference first—no allocation yet.- // We compare the borrowed `&str` with the stored `Option<String>` and- // only allocate (`to_string()`) when a replacement is required.- match &existing.val {- None => {- // Existing is null; new value always wins- let new_val = Some(new_str.to_string());- self.heap.replace_if_better(heap_idx, new_val, map);- }- Some(existing_str) => {- // Compare borrowed strings first- if (!self.desc && new_str < existing_str.as_str())- || (self.desc && new_str > existing_str.as_str())- {- let new_val = Some(new_str.to_string());- self.heap.replace_if_better(heap_idx, new_val, map);- }- // Else: no improvement, no allocation- }- }+ let new_val = (&self.batch).value(row_idx);+ self.heap.replace_if_better(new_val, heap_idx, map);}fn drain(&mut self) -> (ArrayRef, Vec<usize>) {let (vals, map_idxs) = self.heap.drain();- // Use Arrow builders to safely construct arrays from the owned- // `Option<String>` values. Builders avoid needing to maintain- // references to temporary storage.-- // Macro to eliminate duplication across string builder types.- // All three builders share the same interface for append_value,- // append_null, and finish, differing only in their concrete types.- macro_rules! build_string_array {- ($builder_type:ty) => {{- let mut builder = <$builder_type>::new();- for val in vals {- match val {- Some(s) => builder.append_value(&s),- None => builder.append_null(),- }- }- Arc::new(builder.finish())- }};- }-- let arr: ArrayRef = match self.data_type {- DataType::Utf8 => build_string_array!(StringBuilder),- DataType::LargeUtf8 => build_string_array!(LargeStringBuilder),- DataType::Utf8View => build_string_array!(StringViewBuilder),- _ => unreachable!("Unsupported string type: {}", self.data_type),- };- (arr, map_idxs)+ let vals = Arc::new(S::from(vals));+ (vals, map_idxs)}}@@ -326,18 +306,17 @@ impl<VAL: ValueType> TopKHeap<VAL> {desc,capacity: limit,len: 0,- heap: (0..=limit).map(|_| None).collect::<Vec<_>>(),+ heap: (0..=limit).map(|_| HeapItem::default()).collect::<Vec<_>>(),}}pub fn worst_val(&self) -> Option<&VAL> {let root = self.heap.first()?;- let hi = root.as_ref()?;- Some(&hi.val)+ root.is_occupied().then_some(&root.val)}pub fn worst_map_idx(&self) -> usize {- self.heap[0].as_ref().map(|hi| hi.map_idx).unwrap_or(0)+ self.heap[0].map_idx().unwrap_or(0)}pub fn is_full(&self) -> bool {@@ -348,12 +327,14 @@ impl<VAL: ValueType> TopKHeap<VAL> {self.len}- pub fn append_or_replace(+ pub fn append_or_replace<Q>(&mut self,- new_val: VAL,+ new_val: &Q,map_idx: usize,map: &mut Vec<(usize, usize)>,- ) {+ ) where+ Q: ToOwned<Owned = VAL> + ?Sized,+ {if self.is_full() {self.replace_root(new_val, map_idx, map);} else {@@ -361,9 +342,14 @@ impl<VAL: ValueType> TopKHeap<VAL> {}}- fn append(&mut self, new_val: VAL, map_idx: usize, mapper: &mut Vec<(usize, usize)>) {- let hi = HeapItem::new(new_val, map_idx);- self.heap[self.len] = Some(hi);+ fn append<Q>(&mut self, new_val: &Q, map_idx: usize, mapper: &mut Vec<(usize, usize)>)+ where+ Q: ToOwned<Owned = VAL> + ?Sized,+ {+ let hi = &mut self.heap[self.len];+ debug_assert!(hi.is_vacant());+ hi.map_idx = map_idx;+ new_val.clone_into(&mut hi.val);self.heapify_up(self.len, mapper);self.len += 1;}@@ -384,43 +370,48 @@ impl<VAL: ValueType> TopKHeap<VAL> {}pub fn drain(&mut self) -> (Vec<VAL>, Vec<usize>) {- let mut map = Vec::with_capacity(self.len);+ let mut map = Vec::new();let mut vals = Vec::with_capacity(self.len);let mut map_idxs = Vec::with_capacity(self.len);while let Some(worst_hi) = self.pop(&mut map) {vals.push(worst_hi.val);map_idxs.push(worst_hi.map_idx);+ map.clear();}vals.reverse();map_idxs.reverse();(vals, map_idxs)}- fn replace_root(+ fn replace_root<Q>(&mut self,- new_val: VAL,+ new_val: &Q,map_idx: usize,mapper: &mut Vec<(usize, usize)>,- ) {- let hi = self.heap[0].as_mut().expect("No root");- hi.val = new_val;+ ) where+ Q: ToOwned<Owned = VAL> + ?Sized,+ {+ let hi = self.heap[0].as_mut();+ new_val.clone_into(&mut hi.val);hi.map_idx = map_idx;self.heapify_down(0, mapper);}- pub fn replace_if_better(+ pub fn replace_if_better<Q>(&mut self,+ new_val: &Q,heap_idx: usize,- new_val: VAL,mapper: &mut Vec<(usize, usize)>,- ) {- let existing = self.heap[heap_idx].as_mut().expect("Missing heap item");+ ) where+ Q: ToOwned<Owned = VAL> + Comparable<VAL> + ?Sized,+ {+ let existing = self.heap[heap_idx].as_mut();if (!self.desc && new_val.comp(&existing.val) != Ordering::Less)|| (self.desc && new_val.comp(&existing.val) != Ordering::Greater){return;}- existing.val = new_val;+ new_val.clone_into(&mut existing.val);self.heapify_down(heap_idx, mapper);}@@ -428,8 +419,8 @@ impl<VAL: ValueType> TopKHeap<VAL> {let desc = self.desc;while idx != 0 {let parent_idx = (idx - 1) / 2;- let node = self.heap[idx].as_ref().expect("No heap item");- let parent = self.heap[parent_idx].as_ref().expect("No heap item");+ let node = self.heap[idx].as_ref();+ let parent = self.heap[parent_idx].as_ref();if (!desc && node.val.comp(&parent.val) != Ordering::Greater)|| (desc && node.val.comp(&parent.val) != Ordering::Less){@@ -440,40 +431,45 @@ impl<VAL: ValueType> TopKHeap<VAL> {}}+ #[inline]fn swap(&mut self, a_idx: usize, b_idx: usize, mapper: &mut Vec<(usize, usize)>) {self.heap.swap(a_idx, b_idx);- let b_hi = self.heap[b_idx].as_ref().expect("Missing heap entry");- let a_hi = self.heap[a_idx].as_ref().expect("Missing heap entry");+ let b_hi = self.heap[b_idx].as_ref();+ let a_hi = self.heap[a_idx].as_ref();- mapper.push((b_hi.map_idx, b_idx));- mapper.push((a_hi.map_idx, a_idx));+ mapper.extend([(b_hi.map_idx, b_idx), (a_hi.map_idx, a_idx)]);}- fn heapify_down(&mut self, node_idx: usize, mapper: &mut Vec<(usize, usize)>) {- let left_child = node_idx * 2 + 1;+ fn heapify_down(&mut self, mut node_idx: usize, mapper: &mut Vec<(usize, usize)>) {let desc = self.desc;- let entry = self.heap.get(node_idx).expect("Missing node!");- let entry = entry.as_ref().expect("Missing node!");- let mut best_idx = node_idx;- let mut best_val = &entry.val;- for child_idx in left_child..=left_child + 1 {- if let Some(Some(child)) = self.heap.get(child_idx)- && ((!desc && child.val.comp(best_val) == Ordering::Greater)- || (desc && child.val.comp(best_val) == Ordering::Less))- {- best_val = &child.val;- best_idx = child_idx;+ loop {+ let left_child = node_idx * 2 + 1;+ let entry = self.heap[node_idx].as_ref();+ let mut best_idx = node_idx;+ let mut best_val = &entry.val;+ for child_idx in left_child..=left_child + 1 {+ if let Some(child) = self.heap.get(child_idx)+ && child.is_occupied()+ && ((!desc && child.val.comp(best_val) == Ordering::Greater)+ || (desc && child.val.comp(best_val) == Ordering::Less))+ {+ best_val = &child.val;+ best_idx = child_idx;+ }+ }+ if best_val.comp(&entry.val) == Ordering::Equal {+ break;}- }- if best_val.comp(&entry.val) != Ordering::Equal {self.swap(best_idx, node_idx, mapper);- self.heapify_down(best_idx, mapper);+ node_idx = best_idx;}}fn _tree_print(&self, idx: usize, prefix: &str, is_tail: bool, output: &mut String) {- if let Some(Some(hi)) = self.heap.get(idx) {+ if let Some(hi) = self.heap.get(idx)+ && hi.is_occupied()+ {let connector = if idx != 0 {if is_tail { "└── " } else { "├── " }} else {@@ -514,15 +510,6 @@ impl<VAL: ValueType> Display for TopKHeap<VAL> {}}-impl<VAL: ValueType> HeapItem<VAL> {- pub fn new(val: VAL, buk_idx: usize) -> Self {- Self {- val,- map_idx: buk_idx,- }- }-}-impl<VAL: ValueType> Debug for HeapItem<VAL> {fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {f.write_str("bucket=")?;@@ -618,13 +605,6 @@ pub fn new_heap(desc: bool,vt: DataType,) -> Result<Box<dyn ArrowHeap + Send>> {- if matches!(- vt,- DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View- ) {- return Ok(Box::new(StringHeap::new(limit, desc, vt)));- }-macro_rules! downcast_helper {($vt:ty, $d:ident) => {return Ok(Box::new(PrimitiveHeap::<$vt>::new(limit, desc, vt)))@@ -633,6 +613,9 @@ pub fn new_heap(downcast_primitive! {vt => (downcast_helper, vt),+ DataType::Utf8 => return Ok(Box::new(StringHeap::<StringArray>::new(limit, desc))),+ DataType::LargeUtf8 => return Ok(Box::new(StringHeap::<LargeStringArray>::new(limit, desc))),+ DataType::Utf8View => return Ok(Box::new(StringHeap::<StringViewArray>::new(limit, desc))),_ => {}}@@ -651,7 +634,7 @@ mod tests {fn should_append() -> Result<()> {let mut map = vec![];let mut heap = TopKHeap::new(10, false);- heap.append_or_replace(1, 1, &mut map);+ heap.append_or_replace(&1, 1, &mut map);let actual = heap.to_string();assert_snapshot!(actual, @"val=1 idx=0, bucket=1");@@ -664,10 +647,10 @@ mod tests {let mut map = vec![];let mut heap = TopKHeap::new(10, false);- heap.append_or_replace(1, 1, &mut map);+ heap.append_or_replace(&1, 1, &mut map);assert_eq!(map, vec![]);- heap.append_or_replace(2, 2, &mut map);+ heap.append_or_replace(&2, 2, &mut map);assert_eq!(map, vec![(2, 0), (1, 1)]);let actual = heap.to_string();@@ -684,9 +667,9 @@ mod tests {let mut map = vec![];let mut heap = TopKHeap::new(3, false);- heap.append_or_replace(1, 1, &mut map);- heap.append_or_replace(2, 2, &mut map);- heap.append_or_replace(3, 3, &mut map);+ heap.append_or_replace(&1, 1, &mut map);+ heap.append_or_replace(&2, 2, &mut map);+ heap.append_or_replace(&3, 3, &mut map);let actual = heap.to_string();assert_snapshot!(actual, @r"val=3 idx=0, bucket=3@@ -695,7 +678,7 @@ mod tests {");let mut map = vec![];- heap.append_or_replace(0, 0, &mut map);+ heap.append_or_replace(&0, 0, &mut map);let actual = heap.to_string();assert_snapshot!(actual, @r"val=2 idx=0, bucket=2@@ -712,10 +695,10 @@ mod tests {let mut map = vec![];let mut heap = TopKHeap::new(4, false);- heap.append_or_replace(1, 1, &mut map);- heap.append_or_replace(2, 2, &mut map);- heap.append_or_replace(3, 3, &mut map);- heap.append_or_replace(4, 4, &mut map);+ heap.append_or_replace(&1, 1, &mut map);+ heap.append_or_replace(&2, 2, &mut map);+ heap.append_or_replace(&3, 3, &mut map);+ heap.append_or_replace(&4, 4, &mut map);let actual = heap.to_string();assert_snapshot!(actual, @r"val=4 idx=0, bucket=4@@ -725,7 +708,7 @@ mod tests {");let mut map = vec![];- heap.replace_if_better(1, 0, &mut map);+ heap.replace_if_better(&0, 1, &mut map);let actual = heap.to_string();assert_snapshot!(actual, @r"val=4 idx=0, bucket=4@@ -743,8 +726,8 @@ mod tests {let mut map = vec![];let mut heap = TopKHeap::new(10, false);- heap.append_or_replace(1, 1, &mut map);- heap.append_or_replace(2, 2, &mut map);+ heap.append_or_replace(&1, 1, &mut map);+ heap.append_or_replace(&2, 2, &mut map);let actual = heap.to_string();assert_snapshot!(actual, @r"@@ -763,8 +746,8 @@ mod tests {let mut map = vec![];let mut heap = TopKHeap::new(10, false);- heap.append_or_replace(1, 1, &mut map);- heap.append_or_replace(2, 2, &mut map);+ heap.append_or_replace(&1, 1, &mut map);+ heap.append_or_replace(&2, 2, &mut map);let actual = heap.to_string();assert_snapshot!(actual, @r"@@ -779,4 +762,137 @@ mod tests {Ok(())}++ #[test]+ fn should_append_string() -> Result<()> {+ let mut map = vec![];+ let mut heap = TopKHeap::<String>::new(10, false);+ heap.append_or_replace("A", 1, &mut map);++ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"val="A" idx=0, bucket=1"#);++ Ok(())+ }++ #[test]+ fn should_heapify_up_string() -> Result<()> {+ let mut map = vec![];+ let mut heap = TopKHeap::<String>::new(10, false);++ heap.append_or_replace("A", 1, &mut map);+ assert_eq!(map, vec![]);++ heap.append_or_replace("B", 2, &mut map);+ assert_eq!(map, vec![(2, 0), (1, 1)]);++ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"+ val="B" idx=0, bucket=2+ └── val="A" idx=1, bucket=1+ "#);++ Ok(())+ }++ #[test]+ fn should_heapify_down_string() -> Result<()> {+ let mut map = vec![];+ let mut heap = TopKHeap::<String>::new(3, false);++ heap.append_or_replace("B", 1, &mut map);+ heap.append_or_replace("C", 2, &mut map);+ heap.append_or_replace("D", 3, &mut map);+ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"+ val="D" idx=0, bucket=3+ ├── val="B" idx=1, bucket=1+ └── val="C" idx=2, bucket=2+ "#);++ let mut map = vec![];+ heap.append_or_replace("A", 0, &mut map);+ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"+ val="C" idx=0, bucket=2+ ├── val="B" idx=1, bucket=1+ └── val="A" idx=2, bucket=0+ "#);+ assert_eq!(map, vec![(2, 0), (0, 2)]);++ Ok(())+ }++ #[test]+ fn should_replace_string() -> Result<()> {+ let mut map = vec![];+ let mut heap = TopKHeap::<String>::new(4, false);++ heap.append_or_replace("B", 1, &mut map);+ heap.append_or_replace("C", 2, &mut map);+ heap.append_or_replace("D", 3, &mut map);+ heap.append_or_replace("E", 4, &mut map);+ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"+ val="E" idx=0, bucket=4+ ├── val="D" idx=1, bucket=3+ │ └── val="B" idx=3, bucket=1+ └── val="C" idx=2, bucket=2+ "#);++ let mut map = vec![];+ heap.replace_if_better("A", 1, &mut map);+ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"+ val="E" idx=0, bucket=4+ ├── val="B" idx=1, bucket=1+ │ └── val="A" idx=3, bucket=3+ └── val="C" idx=2, bucket=2+ "#);+ assert_eq!(map, vec![(1, 1), (3, 3)]);++ Ok(())+ }++ #[test]+ fn should_find_worst_string() -> Result<()> {+ let mut map = vec![];+ let mut heap = TopKHeap::<String>::new(10, false);++ heap.append_or_replace("A", 1, &mut map);+ heap.append_or_replace("B", 2, &mut map);++ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"+ val="B" idx=0, bucket=2+ └── val="A" idx=1, bucket=1+ "#);++ assert_eq!(heap.worst_val().map(String::as_str), Some("B"));+ assert_eq!(heap.worst_map_idx(), 2);++ Ok(())+ }++ #[test]+ fn should_drain_string() -> Result<()> {+ let mut map = vec![];+ let mut heap = TopKHeap::<String>::new(10, false);++ heap.append_or_replace("A", 1, &mut map);+ heap.append_or_replace("B", 2, &mut map);++ let actual = heap.to_string();+ assert_snapshot!(actual, @r#"+ val="B" idx=0, bucket=2+ └── val="A" idx=1, bucket=1+ "#);++ let (vals, map_idxs) = heap.drain();+ assert_eq!(vals, vec!["A", "B"]);+ assert_eq!(map_idxs, vec![1, 2]);+ assert_eq!(heap.len(), 0);++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs b/datafusion/physical-plan/src/aggregates/topk/priority_map.rsindex f36e0f7628..008580882c 100644--- a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs+++ b/datafusion/physical-plan/src/aggregates/topk/priority_map.rs@@ -30,7 +30,6 @@ pub struct PriorityMap {heap: Box<dyn ArrowHeap + Send>,capacity: usize,mapper: Vec<(usize, usize)>,- val_type: DataType,/// Mirror of the map's all-NULL group count, kept as a plain field so the/// per-row `insert` path can check it without a `dyn` call (measured to/// regress the topk_aggregate benchmarks when read through the trait)@@ -46,10 +45,9 @@ impl PriorityMap {) -> Result<Self> {Ok(Self {map: new_hash_table(capacity, key_type)?,- heap: new_heap(capacity, descending, val_type.clone())?,+ heap: new_heap(capacity, descending, val_type)?,capacity,mapper: Vec::with_capacity(capacity),- val_type,null_count: 0,})}@@ -60,13 +58,14 @@ impl PriorityMap {}pub fn insert(&mut self, row_idx: usize) -> Result<()> {- assert!(self.map.len() <= self.capacity, "Overflow");debug_assert_eq!(self.null_count, 0);// if we're full, and the new val is worse than all our values, just bailif self.heap.is_worse(row_idx) {return Ok(());}+ assert!(self.map.len() <= self.capacity, "Overflow");+self.insert_eligible(row_idx)}@@ -74,10 +73,6 @@ impl PriorityMap {/// separate from [`Self::insert`] so the common no-NULL path does not pay/// for NULL bookkeeping on every row.pub fn insert_with_null_groups(&mut self, row_idx: usize) -> Result<()> {- // valued groups are capped at `capacity`; up to `capacity` additional- // all-NULL groups may be tracked alongside them- assert!(self.map.len() <= 2 * self.capacity, "Overflow");-if self.heap.is_worse(row_idx) {// A group that was registered as all-NULL now has a value that// loses to the current top-k: it can no longer reach the top-k,@@ -87,9 +82,19 @@ impl PriorityMap {}return Ok(());}+ self.null_capacity_assert();+self.insert_eligible(row_idx)}+ #[inline]+ fn null_capacity_assert(&self) {+ // TODO: do debug_assert instead?+ // valued groups are capped at `capacity`; up to `capacity` additional+ // all-NULL groups may be tracked alongside them+ assert!(self.map.len() <= 2 * self.capacity, "Overflow");+ }+fn insert_eligible(&mut self, row_idx: usize) -> Result<()> {let map = &mut self.mapper;@@ -125,8 +130,8 @@ impl PriorityMap {/// must still appear in the aggregation output; such groups all tie on the/// sort key, so tracking up to `capacity` of them preserves top-k semantics.pub fn insert_null(&mut self, row_idx: usize) {- assert!(self.map.len() <= 2 * self.capacity, "Overflow");if self.map.insert_null(row_idx) {+ self.null_capacity_assert();self.null_count += 1;}}@@ -135,12 +140,11 @@ impl PriorityMap {let (vals, mut map_idxs) = self.heap.drain();// Groups whose values are all NULL are tracked in the map only;// append them with a NULL value so they are not lost from the output- let null_idxs = self.map.null_map_idxs();- let vals = if null_idxs.is_empty() {+ let null_count = self.map.null_map_idxs(&mut map_idxs);+ let vals = if null_count == 0 {vals} else {- map_idxs.extend(null_idxs.iter().copied());- let nulls = new_null_array(&self.val_type, null_idxs.len());+ let nulls = new_null_array(self.heap.value_type(), null_count);concat(&[vals.as_ref(), nulls.as_ref()])?};let ids = self.map.take_all(map_idxs);@@ -611,6 +615,31 @@ mod tests {Ok(())}+ #[test]+ fn should_emit_null_id_with_all_null_values() -> Result<()> {+ let ids: ArrayRef = Arc::new(StringArray::from(vec![None, Some("1")]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None, Some(5)]));+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, true)?;+ agg.set_batch(ids, vals);+ agg.insert_null(0);+ agg.insert_with_null_groups(1)?;++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 1 | 5 |+ | | |+ +----------+--------------++ "+ );++ Ok(())+ }+#[test]fn should_cap_null_groups_at_limit() -> Result<()> {let ids: ArrayRef = Arc::new(StringArray::from(vec!["1", "2", "3", "4", "5"]));