Apache DataFusion SQL Query Engine
perf: Concretely typed TopK ArrowHashTable storage (#23609)
Michal Piatkowskic81e7070b706
diff --git a/datafusion-testing b/datafusion-testingindex 13bbae3877..7833a65d5b 160000--- a/datafusion-testing+++ b/datafusion-testing@@ -1 +1 @@-Subproject commit 13bbae38776c2bfbc1fab1be7e7220222d4284bf+Subproject commit 7833a65d5b08be2ca484ea938f471cf01df54e18diff --git a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs b/datafusion/physical-plan/src/aggregates/topk/hash_table.rsindex adc8f8c315..61410893df 100644--- a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs+++ b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs@@ -24,6 +24,7 @@ use arrow::array::{Array, ArrayRef, ArrowPrimitiveType, LargeStringArray, PrimitiveArray, StringArray,StringViewArray, builder::PrimitiveBuilder, cast::AsArray, downcast_primitive,};+use arrow::array::{ArrayAccessor, StringArrayType};use arrow::datatypes::{DataType, i256};use datafusion_common::Result;use datafusion_common::exec_datafusion_err;@@ -120,11 +121,13 @@ pub fn is_supported_hash_key_type(kt: &DataType) -> bool {}// An implementation of ArrowHashTable for String keys-pub struct StringHashTable {- owned: ArrayRef,+pub struct StringHashTable<S: Array>+where+ for<'a> &'a S: StringArrayType<'a>,+{+ owned: S,map: TopKHashTable<Option<String>>,rnd: RandomState,- data_type: DataType,}// An implementation of ArrowHashTable for any `ArrowPrimitiveType` key@@ -132,69 +135,42 @@ struct PrimitiveHashTable<VAL: ArrowPrimitiveType>whereOption<<VAL as ArrowPrimitiveType>::Native>: Comparable,{- owned: ArrayRef,+ owned: PrimitiveArray<VAL>,map: TopKHashTable<Option<VAL::Native>>,rnd: RandomState,- kt: DataType,}-impl StringHashTable {- pub fn new(limit: usize, data_type: DataType) -> Self {- let vals: Vec<&str> = Vec::new();- let owned: ArrayRef = match data_type {- DataType::Utf8 => Arc::new(StringArray::from(vals)),- DataType::Utf8View => Arc::new(StringViewArray::from(vals)),- DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vals)),- _ => panic!("Unsupported data type"),- };-+impl<S> StringHashTable<S>+where+ S: Array + From<Vec<Option<String>>>,+ for<'a> &'a S: StringArrayType<'a>,+{+ pub fn new(limit: usize) -> Self {+ let owned = S::from(Vec::new());Self {owned,map: TopKHashTable::new(limit, limit * 10),rnd: RandomState::default(),- data_type,}}- /// Extracts the string value at the given row index, handling nulls and different string types.- ///- /// Returns `None` if the value is null, otherwise `Some(value.to_string())`.- fn extract_string_value(&self, row_idx: usize) -> Option<String> {- let is_null_and_value = match self.data_type {- DataType::Utf8 => {- let arr = self.owned.as_string::<i32>();- (arr.is_null(row_idx), arr.value(row_idx))- }- DataType::LargeUtf8 => {- let arr = self.owned.as_string::<i64>();- (arr.is_null(row_idx), arr.value(row_idx))- }- DataType::Utf8View => {- let arr = self.owned.as_string_view();- (arr.is_null(row_idx), arr.value(row_idx))- }- _ => panic!("Unsupported data type"),- };-- let (is_null, value) = is_null_and_value;- if is_null {- None- } else {- Some(value.to_string())- }- }-- /// Computes the id and its hash for the given row, for hash table lookups- fn id_and_hash(&self, row_idx: usize) -> (Option<String>, u64) {- let id = self.extract_string_value(row_idx);- let hash = self.rnd.hash_one(id.as_deref());- (id, hash)+ #[inline]+ fn eq_fn(id: Option<&str>) -> impl Fn(&Option<String>) -> bool {+ move |mi| id == mi.as_deref()}}-impl ArrowHashTable for StringHashTable {+impl<S> ArrowHashTable for StringHashTable<S>+where+ S: Array + Clone + From<Vec<Option<String>>> + 'static,+ for<'a> &'a S: StringArrayType<'a>,+{fn set_batch(&mut self, ids: ArrayRef) {- self.owned = ids;+ self.owned = ids+ .as_any()+ .downcast_ref::<S>()+ .expect("Unsupported data type")+ .clone();}fn len(&self) -> usize {@@ -211,12 +187,7 @@ impl ArrowHashTable for StringHashTable {fn take_all(&mut self, indexes: Vec<usize>) -> ArrayRef {let ids = self.map.take_all(indexes);- match self.data_type {- DataType::Utf8 => Arc::new(StringArray::from(ids)),- DataType::LargeUtf8 => Arc::new(LargeStringArray::from(ids)),- DataType::Utf8View => Arc::new(StringViewArray::from(ids)),- _ => unreachable!(),- }+ Arc::new(S::from(ids))}fn find_or_insert(@@ -224,28 +195,30 @@ impl ArrowHashTable for StringHashTable {row_idx: usize,replace_idx: usize,) -> (usize, InsertKind) {- let id = self.extract_string_value(row_idx);-+ let id = some_value(&self.owned, row_idx);// Compute hash and create equality closure for hash table lookup.- let hash = self.rnd.hash_one(id.as_deref());- let id_for_eq = id.clone();- let eq = move |mi: &Option<String>| id_for_eq.as_deref() == mi.as_deref();+ let hash = self.rnd.hash_one(id);// Use entry API to avoid double lookup- self.map.find_or_insert(hash, id, replace_idx, eq)+ self.map.find_or_insert(+ hash,+ id.map(ToOwned::to_owned),+ replace_idx,+ Self::eq_fn(id),+ )}fn insert_null(&mut self, row_idx: usize) -> bool {- let (id, hash) = self.id_and_hash(row_idx);- let id_for_eq = id.clone();- let eq = move |mi: &Option<String>| id_for_eq.as_deref() == mi.as_deref();- self.map.insert_null(hash, id, eq)+ let id = some_value(&self.owned, row_idx);+ let hash = self.rnd.hash_one(id);+ self.map+ .insert_null(hash, id.map(ToOwned::to_owned), Self::eq_fn(id))}fn remove_if_null(&mut self, row_idx: usize) -> bool {- let (id, hash) = self.id_and_hash(row_idx);- let eq = move |mi: &Option<String>| id.as_deref() == mi.as_deref();- self.map.remove_if_null(hash, eq)+ let id = some_value(&self.owned, row_idx);+ let hash = self.rnd.hash_one(id);+ self.map.remove_if_null(hash, Self::eq_fn(id))}fn null_map_idxs(&self) -> Vec<usize> {@@ -255,43 +228,39 @@ impl ArrowHashTable for StringHashTable {impl<VAL: ArrowPrimitiveType> PrimitiveHashTable<VAL>where- Option<<VAL as ArrowPrimitiveType>::Native>: Comparable,- Option<<VAL as ArrowPrimitiveType>::Native>: HashValue,+ Option<<VAL as ArrowPrimitiveType>::Native>: Comparable + HashValue,{pub fn new(limit: usize, kt: DataType) -> Self {- let owned = Arc::new(- PrimitiveArray::<VAL>::builder(0)- .with_data_type(kt.clone())- .finish(),- );+ let owned = PrimitiveArray::<VAL>::builder(0)+ .with_data_type(kt)+ .finish();Self {owned,map: TopKHashTable::new(limit, limit * 10),rnd: RandomState::default(),- kt,}}/// Computes the id and its hash for the given row, for hash table lookups+ #[inline]fn id_and_hash(&self, row_idx: usize) -> (Option<VAL::Native>, u64) {- let ids = self.owned.as_primitive::<VAL>();- let id: Option<VAL::Native> = if ids.is_null(row_idx) {- None- } else {- Some(ids.value(row_idx))- };+ let id: Option<VAL::Native> = some_value(&self.owned, row_idx);let hash: u64 = id.hash(&self.rnd);(id, hash)}++ #[inline]+ fn eq_fn(id: Option<VAL::Native>) -> impl Fn(&Option<VAL::Native>) -> bool {+ move |mi| id == *mi+ }}impl<VAL: ArrowPrimitiveType> ArrowHashTable for PrimitiveHashTable<VAL>where- Option<<VAL as ArrowPrimitiveType>::Native>: Comparable,- Option<<VAL as ArrowPrimitiveType>::Native>: HashValue,+ Option<<VAL as ArrowPrimitiveType>::Native>: Comparable + HashValue,{fn set_batch(&mut self, ids: ArrayRef) {- self.owned = ids;+ self.owned = ids.as_primitive().clone();}fn len(&self) -> usize {@@ -308,8 +277,8 @@ wherefn take_all(&mut self, indexes: Vec<usize>) -> ArrayRef {let ids = self.map.take_all(indexes);- let mut builder: PrimitiveBuilder<VAL> =- PrimitiveArray::builder(ids.len()).with_data_type(self.kt.clone());+ let mut builder: PrimitiveBuilder<VAL> = PrimitiveArray::builder(ids.len())+ .with_data_type(self.owned.data_type().clone());for id in ids.into_iter() {match id {None => builder.append_null(),@@ -325,30 +294,20 @@ whererow_idx: usize,replace_idx: usize,) -> (usize, InsertKind) {- let ids = self.owned.as_primitive::<VAL>();- let id: Option<VAL::Native> = if ids.is_null(row_idx) {- None- } else {- Some(ids.value(row_idx))- };- // Compute hash and create equality closure for hash table lookup.- let hash: u64 = id.hash(&self.rnd);- let eq = |mi: &Option<VAL::Native>| id == *mi;-+ let (id, hash) = self.id_and_hash(row_idx);// Use entry API to avoid double lookup- self.map.find_or_insert(hash, id, replace_idx, eq)+ self.map+ .find_or_insert(hash, id, replace_idx, Self::eq_fn(id))}fn insert_null(&mut self, row_idx: usize) -> bool {let (id, hash) = self.id_and_hash(row_idx);- let eq = move |mi: &Option<VAL::Native>| id == *mi;- self.map.insert_null(hash, id, eq)+ self.map.insert_null(hash, id, Self::eq_fn(id))}fn remove_if_null(&mut self, row_idx: usize) -> bool {let (id, hash) = self.id_and_hash(row_idx);- let eq = move |mi: &Option<VAL::Native>| id == *mi;- self.map.remove_if_null(hash, eq)+ self.map.remove_if_null(hash, Self::eq_fn(id))}fn null_map_idxs(&self) -> Vec<usize> {@@ -474,12 +433,11 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {id: ID,mut eq: impl FnMut(&ID) -> bool,) -> bool {- {- let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);- if self.map.find(hash, eq_fn).is_some() {- return false;- }+ let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);+ if self.map.find(hash, eq_fn).is_some() {+ return false;}+if self.null_count >= self.limit {return false;}@@ -585,6 +543,18 @@ has_integer!(u8, u16, u32, u64);has_integer!(IntervalDayTime, IntervalMonthDayNano);hash_float!(f16, f32, f64);+#[inline]+fn some_value<'a, A>(array: &'a A, index: usize) -> Option<<&'a A as ArrayAccessor>::Item>+where+ &'a A: ArrayAccessor,+{+ if array.is_null(index) {+ None+ } else {+ Some(array.value(index))+ }+}+pub fn new_hash_table(limit: usize,kt: DataType,@@ -597,9 +567,9 @@ pub fn new_hash_table(downcast_primitive! {kt => (downcast_helper, kt),- DataType::Utf8 => return Ok(Box::new(StringHashTable::new(limit, DataType::Utf8))),- DataType::LargeUtf8 => return Ok(Box::new(StringHashTable::new(limit, DataType::LargeUtf8))),- DataType::Utf8View => return Ok(Box::new(StringHashTable::new(limit, DataType::Utf8View))),+ DataType::Utf8 => return Ok(Box::new(StringHashTable::<StringArray>::new(limit))),+ DataType::LargeUtf8 => return Ok(Box::new(StringHashTable::<LargeStringArray>::new(limit))),+ DataType::Utf8View => return Ok(Box::new(StringHashTable::<StringViewArray>::new(limit))),_ => {}}diff --git a/datafusion/physical-plan/src/aggregates/topk/heap.rs b/datafusion/physical-plan/src/aggregates/topk/heap.rsindex ca321cdf99..aef6bb5596 100644--- a/datafusion/physical-plan/src/aggregates/topk/heap.rs+++ b/datafusion/physical-plan/src/aggregates/topk/heap.rs@@ -95,7 +95,7 @@ pub struct PrimitiveHeap<VAL: ArrowPrimitiveType>where<VAL as ArrowPrimitiveType>::Native: Comparable,{- batch: ArrayRef,+ batch: PrimitiveArray<VAL>,heap: TopKHeap<VAL::Native>,desc: bool,data_type: DataType,@@ -106,9 +106,9 @@ where<VAL as ArrowPrimitiveType>::Native: Comparable,{pub fn new(limit: usize, desc: bool, data_type: DataType) -> Self {- let owned: ArrayRef = Arc::new(PrimitiveArray::<VAL>::builder(0).finish());+ let batch = PrimitiveArray::<VAL>::builder(0).finish();Self {- batch: owned,+ batch,heap: TopKHeap::new(limit, desc),desc,data_type,@@ -121,15 +121,14 @@ where<VAL as ArrowPrimitiveType>::Native: Comparable,{fn set_batch(&mut self, vals: ArrayRef) {- self.batch = vals;+ self.batch = PrimitiveArray::from(vals.to_data());}fn is_worse(&self, row_idx: usize) -> bool {if !self.heap.is_full() {return false;}- let vals = self.batch.as_primitive::<VAL>();- let new_val = vals.value(row_idx);+ let new_val = self.batch.value(row_idx);let worst_val = self.heap.worst_val().expect("Missing root");(!self.desc && new_val > *worst_val) || (self.desc && new_val < *worst_val)}@@ -139,8 +138,7 @@ where}fn insert(&mut self, row_idx: usize, map_idx: usize, map: &mut Vec<(usize, usize)>) {- let vals = self.batch.as_primitive::<VAL>();- let new_val = vals.value(row_idx);+ let new_val = self.batch.value(row_idx);self.heap.append_or_replace(new_val, map_idx, map);}@@ -150,8 +148,7 @@ whererow_idx: usize,map: &mut Vec<(usize, usize)>,) {- let vals = self.batch.as_primitive::<VAL>();- let new_val = vals.value(row_idx);+ let new_val = self.batch.value(row_idx);self.heap.replace_if_better(heap_idx, new_val, map);}@@ -443,14 +440,13 @@ impl<VAL: ValueType> TopKHeap<VAL> {}fn swap(&mut self, a_idx: usize, b_idx: usize, mapper: &mut Vec<(usize, usize)>) {- let a_hi = self.heap[a_idx].take().expect("Missing heap entry");- let b_hi = self.heap[b_idx].take().expect("Missing heap entry");+ self.heap.swap(a_idx, b_idx);- mapper.push((a_hi.map_idx, b_idx));- mapper.push((b_hi.map_idx, a_idx));+ let b_hi = self.heap[b_idx].as_ref().expect("Missing heap entry");+ let a_hi = self.heap[a_idx].as_ref().expect("Missing heap entry");- self.heap[a_idx] = Some(b_hi);- self.heap[b_idx] = Some(a_hi);+ mapper.push((b_hi.map_idx, b_idx));+ mapper.push((a_hi.map_idx, a_idx));}fn heapify_down(&mut self, node_idx: usize, mapper: &mut Vec<(usize, usize)>) {