Skip to content

Apache DataFusion SQL Query Engine

perf: Concretely typed TopK ArrowHashTable storage (#23609)

Michal Piatkowskic81e7070b706
diff --git a/datafusion-testing b/datafusion-testing
index 13bbae3877..7833a65d5b 160000
--- a/datafusion-testing
+++ b/datafusion-testing
@@ -1 +1 @@
-Subproject commit 13bbae38776c2bfbc1fab1be7e7220222d4284bf
+Subproject commit 7833a65d5b08be2ca484ea938f471cf01df54e18
diff --git a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs
index adc8f8c315..61410893df 100644
--- a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs
+++ b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs
@@ -24,6 +24,7 @@ use arrow::array::{
Array, ArrayRef, ArrowPrimitiveType, LargeStringArray, PrimitiveArray, StringArray,
StringViewArray, builder::PrimitiveBuilder, cast::AsArray, downcast_primitive,
};
+use arrow::array::{ArrayAccessor, StringArrayType};
use arrow::datatypes::{DataType, i256};
use datafusion_common::Result;
use datafusion_common::exec_datafusion_err;
@@ -120,11 +121,13 @@ pub fn is_supported_hash_key_type(kt: &DataType) -> bool {
}
// An implementation of ArrowHashTable for String keys
-pub struct StringHashTable {
- owned: ArrayRef,
+pub struct StringHashTable<S: Array>
+where
+ for<'a> &'a S: StringArrayType<'a>,
+{
+ owned: S,
map: TopKHashTable<Option<String>>,
rnd: RandomState,
- data_type: DataType,
}
// An implementation of ArrowHashTable for any `ArrowPrimitiveType` key
@@ -132,69 +135,42 @@ struct PrimitiveHashTable<VAL: ArrowPrimitiveType>
where
Option<<VAL as ArrowPrimitiveType>::Native>: Comparable,
{
- owned: ArrayRef,
+ owned: PrimitiveArray<VAL>,
map: TopKHashTable<Option<VAL::Native>>,
rnd: RandomState,
- kt: DataType,
}
-impl StringHashTable {
- pub fn new(limit: usize, data_type: DataType) -> Self {
- let vals: Vec<&str> = Vec::new();
- let owned: ArrayRef = match data_type {
- DataType::Utf8 => Arc::new(StringArray::from(vals)),
- DataType::Utf8View => Arc::new(StringViewArray::from(vals)),
- DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vals)),
- _ => panic!("Unsupported data type"),
- };
-
+impl<S> StringHashTable<S>
+where
+ S: Array + From<Vec<Option<String>>>,
+ for<'a> &'a S: StringArrayType<'a>,
+{
+ pub fn new(limit: usize) -> Self {
+ let owned = S::from(Vec::new());
Self {
owned,
map: TopKHashTable::new(limit, limit * 10),
rnd: RandomState::default(),
- data_type,
}
}
- /// Extracts the string value at the given row index, handling nulls and different string types.
- ///
- /// Returns `None` if the value is null, otherwise `Some(value.to_string())`.
- fn extract_string_value(&self, row_idx: usize) -> Option<String> {
- let is_null_and_value = match self.data_type {
- DataType::Utf8 => {
- let arr = self.owned.as_string::<i32>();
- (arr.is_null(row_idx), arr.value(row_idx))
- }
- DataType::LargeUtf8 => {
- let arr = self.owned.as_string::<i64>();
- (arr.is_null(row_idx), arr.value(row_idx))
- }
- DataType::Utf8View => {
- let arr = self.owned.as_string_view();
- (arr.is_null(row_idx), arr.value(row_idx))
- }
- _ => panic!("Unsupported data type"),
- };
-
- let (is_null, value) = is_null_and_value;
- if is_null {
- None
- } else {
- Some(value.to_string())
- }
- }
-
- /// Computes the id and its hash for the given row, for hash table lookups
- fn id_and_hash(&self, row_idx: usize) -> (Option<String>, u64) {
- let id = self.extract_string_value(row_idx);
- let hash = self.rnd.hash_one(id.as_deref());
- (id, hash)
+ #[inline]
+ fn eq_fn(id: Option<&str>) -> impl Fn(&Option<String>) -> bool {
+ move |mi| id == mi.as_deref()
}
}
-impl ArrowHashTable for StringHashTable {
+impl<S> ArrowHashTable for StringHashTable<S>
+where
+ S: Array + Clone + From<Vec<Option<String>>> + 'static,
+ for<'a> &'a S: StringArrayType<'a>,
+{
fn set_batch(&mut self, ids: ArrayRef) {
- self.owned = ids;
+ self.owned = ids
+ .as_any()
+ .downcast_ref::<S>()
+ .expect("Unsupported data type")
+ .clone();
}
fn len(&self) -> usize {
@@ -211,12 +187,7 @@ impl ArrowHashTable for StringHashTable {
fn take_all(&mut self, indexes: Vec<usize>) -> ArrayRef {
let ids = self.map.take_all(indexes);
- match self.data_type {
- DataType::Utf8 => Arc::new(StringArray::from(ids)),
- DataType::LargeUtf8 => Arc::new(LargeStringArray::from(ids)),
- DataType::Utf8View => Arc::new(StringViewArray::from(ids)),
- _ => unreachable!(),
- }
+ Arc::new(S::from(ids))
}
fn find_or_insert(
@@ -224,28 +195,30 @@ impl ArrowHashTable for StringHashTable {
row_idx: usize,
replace_idx: usize,
) -> (usize, InsertKind) {
- let id = self.extract_string_value(row_idx);
-
+ let id = some_value(&self.owned, row_idx);
// Compute hash and create equality closure for hash table lookup.
- let hash = self.rnd.hash_one(id.as_deref());
- let id_for_eq = id.clone();
- let eq = move |mi: &Option<String>| id_for_eq.as_deref() == mi.as_deref();
+ let hash = self.rnd.hash_one(id);
// Use entry API to avoid double lookup
- self.map.find_or_insert(hash, id, replace_idx, eq)
+ self.map.find_or_insert(
+ hash,
+ id.map(ToOwned::to_owned),
+ replace_idx,
+ Self::eq_fn(id),
+ )
}
fn insert_null(&mut self, row_idx: usize) -> bool {
- let (id, hash) = self.id_and_hash(row_idx);
- let id_for_eq = id.clone();
- let eq = move |mi: &Option<String>| id_for_eq.as_deref() == mi.as_deref();
- self.map.insert_null(hash, id, eq)
+ let id = some_value(&self.owned, row_idx);
+ let hash = self.rnd.hash_one(id);
+ self.map
+ .insert_null(hash, id.map(ToOwned::to_owned), Self::eq_fn(id))
}
fn remove_if_null(&mut self, row_idx: usize) -> bool {
- let (id, hash) = self.id_and_hash(row_idx);
- let eq = move |mi: &Option<String>| id.as_deref() == mi.as_deref();
- self.map.remove_if_null(hash, eq)
+ let id = some_value(&self.owned, row_idx);
+ let hash = self.rnd.hash_one(id);
+ self.map.remove_if_null(hash, Self::eq_fn(id))
}
fn null_map_idxs(&self) -> Vec<usize> {
@@ -255,43 +228,39 @@ impl ArrowHashTable for StringHashTable {
impl<VAL: ArrowPrimitiveType> PrimitiveHashTable<VAL>
where
- Option<<VAL as ArrowPrimitiveType>::Native>: Comparable,
- Option<<VAL as ArrowPrimitiveType>::Native>: HashValue,
+ Option<<VAL as ArrowPrimitiveType>::Native>: Comparable + HashValue,
{
pub fn new(limit: usize, kt: DataType) -> Self {
- let owned = Arc::new(
- PrimitiveArray::<VAL>::builder(0)
- .with_data_type(kt.clone())
- .finish(),
- );
+ let owned = PrimitiveArray::<VAL>::builder(0)
+ .with_data_type(kt)
+ .finish();
Self {
owned,
map: TopKHashTable::new(limit, limit * 10),
rnd: RandomState::default(),
- kt,
}
}
/// Computes the id and its hash for the given row, for hash table lookups
+ #[inline]
fn id_and_hash(&self, row_idx: usize) -> (Option<VAL::Native>, u64) {
- let ids = self.owned.as_primitive::<VAL>();
- let id: Option<VAL::Native> = if ids.is_null(row_idx) {
- None
- } else {
- Some(ids.value(row_idx))
- };
+ let id: Option<VAL::Native> = some_value(&self.owned, row_idx);
let hash: u64 = id.hash(&self.rnd);
(id, hash)
}
+
+ #[inline]
+ fn eq_fn(id: Option<VAL::Native>) -> impl Fn(&Option<VAL::Native>) -> bool {
+ move |mi| id == *mi
+ }
}
impl<VAL: ArrowPrimitiveType> ArrowHashTable for PrimitiveHashTable<VAL>
where
- Option<<VAL as ArrowPrimitiveType>::Native>: Comparable,
- Option<<VAL as ArrowPrimitiveType>::Native>: HashValue,
+ Option<<VAL as ArrowPrimitiveType>::Native>: Comparable + HashValue,
{
fn set_batch(&mut self, ids: ArrayRef) {
- self.owned = ids;
+ self.owned = ids.as_primitive().clone();
}
fn len(&self) -> usize {
@@ -308,8 +277,8 @@ where
fn take_all(&mut self, indexes: Vec<usize>) -> ArrayRef {
let ids = self.map.take_all(indexes);
- let mut builder: PrimitiveBuilder<VAL> =
- PrimitiveArray::builder(ids.len()).with_data_type(self.kt.clone());
+ let mut builder: PrimitiveBuilder<VAL> = PrimitiveArray::builder(ids.len())
+ .with_data_type(self.owned.data_type().clone());
for id in ids.into_iter() {
match id {
None => builder.append_null(),
@@ -325,30 +294,20 @@ where
row_idx: usize,
replace_idx: usize,
) -> (usize, InsertKind) {
- let ids = self.owned.as_primitive::<VAL>();
- let id: Option<VAL::Native> = if ids.is_null(row_idx) {
- None
- } else {
- Some(ids.value(row_idx))
- };
- // Compute hash and create equality closure for hash table lookup.
- let hash: u64 = id.hash(&self.rnd);
- let eq = |mi: &Option<VAL::Native>| id == *mi;
-
+ let (id, hash) = self.id_and_hash(row_idx);
// Use entry API to avoid double lookup
- self.map.find_or_insert(hash, id, replace_idx, eq)
+ self.map
+ .find_or_insert(hash, id, replace_idx, Self::eq_fn(id))
}
fn insert_null(&mut self, row_idx: usize) -> bool {
let (id, hash) = self.id_and_hash(row_idx);
- let eq = move |mi: &Option<VAL::Native>| id == *mi;
- self.map.insert_null(hash, id, eq)
+ self.map.insert_null(hash, id, Self::eq_fn(id))
}
fn remove_if_null(&mut self, row_idx: usize) -> bool {
let (id, hash) = self.id_and_hash(row_idx);
- let eq = move |mi: &Option<VAL::Native>| id == *mi;
- self.map.remove_if_null(hash, eq)
+ self.map.remove_if_null(hash, Self::eq_fn(id))
}
fn null_map_idxs(&self) -> Vec<usize> {
@@ -474,12 +433,11 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {
id: ID,
mut eq: impl FnMut(&ID) -> bool,
) -> bool {
- {
- let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);
- if self.map.find(hash, eq_fn).is_some() {
- return false;
- }
+ let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);
+ if self.map.find(hash, eq_fn).is_some() {
+ return false;
}
+
if self.null_count >= self.limit {
return false;
}
@@ -585,6 +543,18 @@ has_integer!(u8, u16, u32, u64);
has_integer!(IntervalDayTime, IntervalMonthDayNano);
hash_float!(f16, f32, f64);
+#[inline]
+fn some_value<'a, A>(array: &'a A, index: usize) -> Option<<&'a A as ArrayAccessor>::Item>
+where
+ &'a A: ArrayAccessor,
+{
+ if array.is_null(index) {
+ None
+ } else {
+ Some(array.value(index))
+ }
+}
+
pub fn new_hash_table(
limit: usize,
kt: DataType,
@@ -597,9 +567,9 @@ pub fn new_hash_table(
downcast_primitive! {
kt => (downcast_helper, kt),
- DataType::Utf8 => return Ok(Box::new(StringHashTable::new(limit, DataType::Utf8))),
- DataType::LargeUtf8 => return Ok(Box::new(StringHashTable::new(limit, DataType::LargeUtf8))),
- DataType::Utf8View => return Ok(Box::new(StringHashTable::new(limit, DataType::Utf8View))),
+ DataType::Utf8 => return Ok(Box::new(StringHashTable::<StringArray>::new(limit))),
+ DataType::LargeUtf8 => return Ok(Box::new(StringHashTable::<LargeStringArray>::new(limit))),
+ DataType::Utf8View => return Ok(Box::new(StringHashTable::<StringViewArray>::new(limit))),
_ => {}
}
diff --git a/datafusion/physical-plan/src/aggregates/topk/heap.rs b/datafusion/physical-plan/src/aggregates/topk/heap.rs
index ca321cdf99..aef6bb5596 100644
--- a/datafusion/physical-plan/src/aggregates/topk/heap.rs
+++ b/datafusion/physical-plan/src/aggregates/topk/heap.rs
@@ -95,7 +95,7 @@ pub struct PrimitiveHeap<VAL: ArrowPrimitiveType>
where
<VAL as ArrowPrimitiveType>::Native: Comparable,
{
- batch: ArrayRef,
+ batch: PrimitiveArray<VAL>,
heap: TopKHeap<VAL::Native>,
desc: bool,
data_type: DataType,
@@ -106,9 +106,9 @@ where
<VAL as ArrowPrimitiveType>::Native: Comparable,
{
pub fn new(limit: usize, desc: bool, data_type: DataType) -> Self {
- let owned: ArrayRef = Arc::new(PrimitiveArray::<VAL>::builder(0).finish());
+ let batch = PrimitiveArray::<VAL>::builder(0).finish();
Self {
- batch: owned,
+ batch,
heap: TopKHeap::new(limit, desc),
desc,
data_type,
@@ -121,15 +121,14 @@ where
<VAL as ArrowPrimitiveType>::Native: Comparable,
{
fn set_batch(&mut self, vals: ArrayRef) {
- self.batch = vals;
+ self.batch = PrimitiveArray::from(vals.to_data());
}
fn is_worse(&self, row_idx: usize) -> bool {
if !self.heap.is_full() {
return false;
}
- let vals = self.batch.as_primitive::<VAL>();
- let new_val = vals.value(row_idx);
+ let new_val = self.batch.value(row_idx);
let worst_val = self.heap.worst_val().expect("Missing root");
(!self.desc && new_val > *worst_val) || (self.desc && new_val < *worst_val)
}
@@ -139,8 +138,7 @@ where
}
fn insert(&mut self, row_idx: usize, map_idx: usize, map: &mut Vec<(usize, usize)>) {
- let vals = self.batch.as_primitive::<VAL>();
- let new_val = vals.value(row_idx);
+ let new_val = self.batch.value(row_idx);
self.heap.append_or_replace(new_val, map_idx, map);
}
@@ -150,8 +148,7 @@ where
row_idx: usize,
map: &mut Vec<(usize, usize)>,
) {
- let vals = self.batch.as_primitive::<VAL>();
- let new_val = vals.value(row_idx);
+ let new_val = self.batch.value(row_idx);
self.heap.replace_if_better(heap_idx, new_val, map);
}
@@ -443,14 +440,13 @@ impl<VAL: ValueType> TopKHeap<VAL> {
}
fn swap(&mut self, a_idx: usize, b_idx: usize, mapper: &mut Vec<(usize, usize)>) {
- let a_hi = self.heap[a_idx].take().expect("Missing heap entry");
- let b_hi = self.heap[b_idx].take().expect("Missing heap entry");
+ self.heap.swap(a_idx, b_idx);
- mapper.push((a_hi.map_idx, b_idx));
- mapper.push((b_hi.map_idx, a_idx));
+ let b_hi = self.heap[b_idx].as_ref().expect("Missing heap entry");
+ let a_hi = self.heap[a_idx].as_ref().expect("Missing heap entry");
- self.heap[a_idx] = Some(b_hi);
- self.heap[b_idx] = Some(a_hi);
+ mapper.push((b_hi.map_idx, b_idx));
+ mapper.push((a_hi.map_idx, a_idx));
}
fn heapify_down(&mut self, node_idx: usize, mapper: &mut Vec<(usize, usize)>) {