Apache DataFusion SQL Query Engine
fix: TopK aggregation drops groups whose MIN/MAX value is NULL (#23684)
kidf8b9ed81f0dc
diff --git a/datafusion/physical-optimizer/src/topk_aggregation.rs b/datafusion/physical-optimizer/src/topk_aggregation.rsindex e1779c04a6..0eddb5d550 100644--- a/datafusion/physical-optimizer/src/topk_aggregation.rs+++ b/datafusion/physical-optimizer/src/topk_aggregation.rs@@ -46,6 +46,7 @@ impl TopKAggregation {aggr: &AggregateExec,order_by: &str,order_desc: bool,+ nulls_first: bool,limit: usize,) -> Option<Arc<dyn ExecutionPlan>> {// Current only support single group key@@ -66,6 +67,26 @@ impl TopKAggregation {// Check if this is ordering by an aggregate function (MIN/MAX)if let Some((field, desc)) = aggr.get_minmax_desc() {+ // A nullable MIN/MAX starts as NULL and becomes non-NULL when the+ // group sees its first value. With NULLS FIRST that transition+ // worsens the group's rank, so a bounded aggregation cannot safely+ // discard other NULL groups. Use regular aggregation for exact+ // results. Non-nullable inputs never take this transition and can+ // still use TopK.+ let input_nullable = aggr+ .aggr_expr()+ .iter()+ .exactly_one()+ .ok()?+ .expressions()+ .into_iter()+ .exactly_one()+ .ok()?+ .nullable(aggr.input_schema.as_ref())+ .ok()?;+ if nulls_first && input_nullable {+ return None;+ }// ensure the sort direction matches aggregate functionif desc != order_desc {return None;@@ -100,6 +121,7 @@ impl TopKAggregation {let order = sort.properties().output_ordering()?;let order = order.iter().exactly_one().ok()?;let order_desc = order.options.descending;+ let nulls_first = order.options.nulls_first;let order = order.expr.downcast_ref::<Column>()?;let mut cur_col_name = order.name().to_string();let limit = sort.fetch()?;@@ -111,7 +133,13 @@ impl TopKAggregation {}if let Some(aggr) = plan.downcast_ref::<AggregateExec>() {// either we run into an Aggregate and transform it- match Self::transform_agg(aggr, &cur_col_name, order_desc, limit) {+ match Self::transform_agg(+ aggr,+ &cur_col_name,+ order_desc,+ nulls_first,+ limit,+ ) {None => cardinality_preserved = false,Some(plan) => return Ok(Transformed::yes(plan)),}diff --git a/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rsindex 97f4662c11..193fdba4b0 100644--- a/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rs+++ b/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rs@@ -149,11 +149,26 @@ impl GroupedTopKAggregateStream {if has_nulls && self.is_group_by_only() {self.null_group_seen = true;}+ // Keep the common no-NULL path free of NULL bookkeeping. Once a NULL+ // group exists, use the NULL-aware path until it has been resolved.+ let track_null_groups = !self.is_group_by_only()+ && (has_nulls || self.priority_map.has_null_groups());for row_idx in 0..len {if has_nulls && vals.is_null(row_idx) {+ // MIN/MAX ignore NULL inputs, but a group whose values are all+ // NULL must still be emitted with a NULL aggregate value, so+ // track it. (GROUP BY-only aggregations handle NULL group keys+ // via `null_group_seen` instead.)+ if !self.is_group_by_only() {+ self.priority_map.insert_null(row_idx);+ }continue;}- self.priority_map.insert(row_idx)?;+ if track_null_groups {+ self.priority_map.insert_with_null_groups(row_idx)?;+ } else {+ self.priority_map.insert(row_idx)?;+ }}Ok(())}diff --git a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs b/datafusion/physical-plan/src/aggregates/topk/hash_table.rsindex 694780f085..adc8f8c315 100644--- a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs+++ b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs@@ -39,6 +39,11 @@ pub trait KeyType: Clone + Comparable + Debug {}impl<T> KeyType for T where T: Clone + Comparable + Debug {}+/// `heap_idx` assigned to groups whose aggregate values are all NULL. Such+/// groups are tracked in the hash table only (they never enter the heap), so+/// they can be emitted with a NULL aggregate value at the end.+const NULL_HEAP_IDX: usize = usize::MAX;+/// An entry in our hash table that:/// 1. memoizes the hash/// 2. contains the key (ID)@@ -57,10 +62,25 @@ struct TopKHashTable<ID: KeyType> {map: HashTable<usize>,// Store the actual items separately to allow for index-based accessstore: Vec<Option<HashTableItem<ID>>>,- // Free index in the store for reuse- free_index: Option<usize>,+ // Free indexes in the store for reuse+ free_indices: Vec<usize>,// The maximum number of entries allowedlimit: usize,+ // Number of entries registered as all-NULL (heap_idx == NULL_HEAP_IDX)+ null_count: usize,+}++/// Outcome of [`ArrowHashTable::find_or_insert`], letting the caller keep its+/// own all-NULL group accounting in sync without an extra lookup.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum InsertKind {+ /// The group already existed as a valued group+ Existing,+ /// The group was newly inserted as a valued group+ New,+ /// The group was registered as all-NULL and has now been converted into a+ /// valued group+ ReplacedNull,}/// An interface to hide the generic type signature of TopKHashTable behind arrow arrays@@ -70,7 +90,20 @@ pub trait ArrowHashTable {fn update_heap_idx(&mut self, mapper: &[(usize, usize)]);fn heap_idx_at(&self, map_idx: usize) -> usize;fn take_all(&mut self, indexes: Vec<usize>) -> ArrayRef;- fn find_or_insert(&mut self, row_idx: usize, replace_idx: usize) -> (usize, bool);+ fn find_or_insert(+ &mut self,+ row_idx: usize,+ replace_idx: usize,+ ) -> (usize, InsertKind);+ /// Register the group at `row_idx` as all-NULL. Returns true if it was+ /// newly registered; false if the group is already tracked or the NULL+ /// group limit has been reached.+ fn insert_null(&mut self, row_idx: usize) -> bool;+ /// Remove the group at `row_idx` if it is registered as all-NULL. Returns+ /// true if a NULL registration was removed.+ fn remove_if_null(&mut self, row_idx: usize) -> bool;+ /// Store indexes of all groups registered as all-NULL+ fn null_map_idxs(&self) -> Vec<usize>;}/// Returns true if the given data type can be used as a top-K aggregation hash key.@@ -150,6 +183,13 @@ impl StringHashTable {Some(value.to_string())}}++ /// Computes the id and its hash for the given row, for hash table lookups+ fn id_and_hash(&self, row_idx: usize) -> (Option<String>, u64) {+ let id = self.extract_string_value(row_idx);+ let hash = self.rnd.hash_one(id.as_deref());+ (id, hash)+ }}impl ArrowHashTable for StringHashTable {@@ -179,7 +219,11 @@ impl ArrowHashTable for StringHashTable {}}- fn find_or_insert(&mut self, row_idx: usize, replace_idx: usize) -> (usize, bool) {+ fn find_or_insert(+ &mut self,+ row_idx: usize,+ replace_idx: usize,+ ) -> (usize, InsertKind) {let id = self.extract_string_value(row_idx);// Compute hash and create equality closure for hash table lookup.@@ -190,6 +234,23 @@ impl ArrowHashTable for StringHashTable {// Use entry API to avoid double lookupself.map.find_or_insert(hash, id, replace_idx, eq)}++ fn insert_null(&mut self, row_idx: usize) -> bool {+ let (id, hash) = self.id_and_hash(row_idx);+ let id_for_eq = id.clone();+ let eq = move |mi: &Option<String>| id_for_eq.as_deref() == mi.as_deref();+ self.map.insert_null(hash, id, eq)+ }++ fn remove_if_null(&mut self, row_idx: usize) -> bool {+ let (id, hash) = self.id_and_hash(row_idx);+ let eq = move |mi: &Option<String>| id.as_deref() == mi.as_deref();+ self.map.remove_if_null(hash, eq)+ }++ fn null_map_idxs(&self) -> Vec<usize> {+ self.map.null_map_idxs()+ }}impl<VAL: ArrowPrimitiveType> PrimitiveHashTable<VAL>@@ -210,6 +271,18 @@ wherekt,}}++ /// Computes the id and its hash for the given row, for hash table lookups+ fn id_and_hash(&self, row_idx: usize) -> (Option<VAL::Native>, u64) {+ let ids = self.owned.as_primitive::<VAL>();+ let id: Option<VAL::Native> = if ids.is_null(row_idx) {+ None+ } else {+ Some(ids.value(row_idx))+ };+ let hash: u64 = id.hash(&self.rnd);+ (id, hash)+ }}impl<VAL: ArrowPrimitiveType> ArrowHashTable for PrimitiveHashTable<VAL>@@ -247,7 +320,11 @@ whereArc::new(ids)}- fn find_or_insert(&mut self, row_idx: usize, replace_idx: usize) -> (usize, bool) {+ fn find_or_insert(+ &mut self,+ row_idx: usize,+ replace_idx: usize,+ ) -> (usize, InsertKind) {let ids = self.owned.as_primitive::<VAL>();let id: Option<VAL::Native> = if ids.is_null(row_idx) {None@@ -261,6 +338,22 @@ where// Use entry API to avoid double lookupself.map.find_or_insert(hash, id, replace_idx, eq)}++ fn insert_null(&mut self, row_idx: usize) -> bool {+ let (id, hash) = self.id_and_hash(row_idx);+ let eq = move |mi: &Option<VAL::Native>| id == *mi;+ self.map.insert_null(hash, id, eq)+ }++ fn remove_if_null(&mut self, row_idx: usize) -> bool {+ let (id, hash) = self.id_and_hash(row_idx);+ let eq = move |mi: &Option<VAL::Native>| id == *mi;+ self.map.remove_if_null(hash, eq)+ }++ fn null_map_idxs(&self) -> Vec<usize> {+ self.map.null_map_idxs()+ }}use hashbrown::hash_table::Entry;@@ -269,8 +362,9 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {Self {map: HashTable::with_capacity(capacity),store: Vec::with_capacity(capacity),- free_index: None,+ free_indices: Vec::new(),limit,+ null_count: 0,}}@@ -278,25 +372,33 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {self.store[map_idx].as_ref().unwrap().heap_idx}- pub fn remove_if_full(&mut self, replace_idx: usize) -> usize {- if self.map.len() >= self.limit {- let item_to_remove = self.store[replace_idx].as_ref().unwrap();- let hash = item_to_remove.hash;- let id_to_remove = &item_to_remove.id;-- let eq = |&idx: &usize| self.store[idx].as_ref().unwrap().id == *id_to_remove;- let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;- match self.map.entry(hash, eq, hasher) {- Entry::Occupied(entry) => {- let (removed_idx, _) = entry.remove();- self.store[removed_idx] = None;- self.free_index = Some(removed_idx);- }- Entry::Vacant(_) => unreachable!(),+ /// Remove the entry stored at `map_idx`, freeing its store slot for reuse+ fn remove_at(&mut self, map_idx: usize) {+ let item_to_remove = self.store[map_idx].as_ref().unwrap();+ let hash = item_to_remove.hash;+ let id_to_remove = &item_to_remove.id;++ let eq = |&idx: &usize| self.store[idx].as_ref().unwrap().id == *id_to_remove;+ let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;+ match self.map.entry(hash, eq, hasher) {+ Entry::Occupied(entry) => {+ let (removed_idx, _) = entry.remove();+ self.store[removed_idx] = None;+ self.free_indices.push(removed_idx);}+ Entry::Vacant(_) => unreachable!(),+ }+ }++ pub fn remove_if_full(&mut self, replace_idx: usize) -> usize {+ // All-NULL groups are tracked outside the heap, so only valued+ // groups count towards the limit here+ let valued_len = self.map.len() - self.null_count;+ if valued_len >= self.limit {+ self.remove_at(replace_idx);0 // if full, always replace top node} else {- self.map.len() // if we're not full, always append to end+ valued_len // if we're not full, always append to end}}@@ -307,7 +409,8 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {}/// Find an existing entry or insert a new one, avoiding double hash table lookup.- /// Returns (map_idx, is_new) where is_new indicates if this was a new insertion.+ /// Returns (map_idx, kind) where kind describes whether the group already+ /// existed, was newly inserted, or was converted from an all-NULL group./// If inserting a new entry and the table is full, replaces the entry at replace_idx.pub fn find_or_insert(&mut self,@@ -315,19 +418,28 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {id: ID,replace_idx: usize,mut eq: impl FnMut(&ID) -> bool,- ) -> (usize, bool) {+ ) -> (usize, InsertKind) {// Check if entry exists - this is the only hash table lookup+ let mut replaced_null = false;{let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);if let Some(&map_idx) = self.map.find(hash, eq_fn) {- return (map_idx, false);+ if self.store[map_idx].as_ref().unwrap().heap_idx == NULL_HEAP_IDX {+ // This group was registered as all-NULL but now produced a+ // value: unregister it so it is inserted as a valued group+ self.remove_at(map_idx);+ self.null_count -= 1;+ replaced_null = true;+ } else {+ return (map_idx, InsertKind::Existing);+ }}}// Entry doesn't exist - compute heap_idx and prepare itemlet heap_idx = self.remove_if_full(replace_idx);let mi = HashTableItem::new(hash, id, heap_idx);- let store_idx = if let Some(idx) = self.free_index.take() {+ let store_idx = if let Some(idx) = self.free_indices.pop() {self.store[idx] = Some(mi);idx} else {@@ -343,7 +455,80 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {// Insert without checking again since we already confirmed it doesn't existself.map.insert_unique(hash, store_idx, hasher);- (store_idx, true)+ let kind = if replaced_null {+ InsertKind::ReplacedNull+ } else {+ InsertKind::New+ };+ (store_idx, kind)+ }++ /// Register a group whose aggregate values are all NULL, unless it is+ /// already tracked. NULL groups are stored with a sentinel `heap_idx` and+ /// never enter the heap. At most `limit` NULL groups are tracked: they all+ /// tie on the sort key, so any `limit` of them is a valid top-k superset.+ /// Returns true if the group was newly registered.+ pub fn insert_null(+ &mut self,+ hash: u64,+ id: ID,+ mut eq: impl FnMut(&ID) -> bool,+ ) -> bool {+ {+ let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);+ if self.map.find(hash, eq_fn).is_some() {+ return false;+ }+ }+ if self.null_count >= self.limit {+ return false;+ }++ let mi = HashTableItem::new(hash, id, NULL_HEAP_IDX);+ let store_idx = if let Some(idx) = self.free_indices.pop() {+ self.store[idx] = Some(mi);+ idx+ } else {+ self.store.push(Some(mi));+ self.store.len() - 1+ };++ let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;+ if self.map.len() == self.map.capacity() {+ self.map.reserve(self.limit, hasher);+ }+ self.map.insert_unique(hash, store_idx, hasher);+ self.null_count += 1;+ true+ }++ /// Remove the given group if it is registered as all-NULL. Used when an+ /// all-NULL group produces a value that loses to the current top-k: the+ /// group can no longer reach the top-k, but it must not be emitted with a+ /// NULL value either. Returns true if a NULL registration was removed.+ pub fn remove_if_null(&mut self, hash: u64, mut eq: impl FnMut(&ID) -> bool) -> bool {+ let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);+ if let Some(&map_idx) = self.map.find(hash, eq_fn)+ && self.store[map_idx].as_ref().unwrap().heap_idx == NULL_HEAP_IDX+ {+ self.remove_at(map_idx);+ self.null_count -= 1;+ return true;+ }+ false+ }++ /// Store indexes of all groups registered as all-NULL+ pub fn null_map_idxs(&self) -> Vec<usize> {+ self.store+ .iter()+ .enumerate()+ .filter_map(|(idx, item)| {+ item.as_ref()+ .filter(|item| item.heap_idx == NULL_HEAP_IDX)+ .map(|_| idx)+ })+ .collect()}pub fn len(&self) -> usize {@@ -357,7 +542,8 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {.collect();self.map.clear();self.store.clear();- self.free_index = None;+ self.free_indices.clear();+ self.null_count = 0;ids}}@@ -453,9 +639,9 @@ mod tests {for (heap_idx, id) in ["1", "2", "3", "4", "5"].iter().enumerate() {let value = Some(id.to_string());let hash = heap_idx as u64;- let (map_idx, is_new) =+ let (map_idx, kind) =map.find_or_insert(hash, value.clone(), heap_idx, |v| *v == value);- assert!(is_new, "Entry should be new");+ assert_eq!(kind, InsertKind::New, "Entry should be new");heap_to_map.insert(heap_idx, map_idx);}@@ -477,4 +663,65 @@ mod tests {Ok(())}++ #[test]+ fn should_track_null_groups() -> Result<()> {+ let mut map = TopKHashTable::<Option<String>>::new(2, 10);++ let a = Some("a".to_string());+ let b = Some("b".to_string());+ let c = Some("c".to_string());++ // register two all-NULL groups; the third exceeds the NULL group limit+ assert!(map.insert_null(100, a.clone(), |v| *v == a));+ assert!(map.insert_null(200, b.clone(), |v| *v == b));+ assert!(!map.insert_null(300, c.clone(), |v| *v == c));+ // re-registering an existing NULL group is a no-op+ assert!(!map.insert_null(100, a.clone(), |v| *v == a));+ assert_eq!(map.null_count, 2);+ assert_eq!(map.null_map_idxs(), vec![0, 1]);++ // a valued insert for a NULL group converts it to a valued group+ let (map_idx, kind) = map.find_or_insert(200, b.clone(), 0, |v| *v == b);+ assert_eq!(kind, InsertKind::ReplacedNull, "NULL group should convert");+ assert_eq!(map.heap_idx_at(map_idx), 0, "Heap should append at 0");+ assert_eq!(map.null_count, 1);+ assert_eq!(map.null_map_idxs(), vec![0]);++ // remove the remaining NULL group; removing twice is a no-op+ map.remove_if_null(100, |v| *v == a);+ assert_eq!(map.null_count, 0);+ assert!(map.null_map_idxs().is_empty());+ map.remove_if_null(100, |v| *v == a);+ // removing a valued group via remove_if_null is a no-op+ map.remove_if_null(200, |v| *v == b);+ assert_eq!(map.len(), 1);++ Ok(())+ }++ #[test]+ fn should_reuse_all_freed_store_slots() -> Result<()> {+ let mut map = TopKHashTable::<Option<String>>::new(1, 10);++ let a = Some("a".to_string());+ let b = Some("b".to_string());+ let c = Some("c".to_string());++ let (b_idx, kind) = map.find_or_insert(100, b.clone(), 0, |v| *v == b);+ assert_eq!(kind, InsertKind::New);+ assert!(map.insert_null(200, a.clone(), |v| *v == a));++ // Converting a NULL group while the valued heap is full frees two+ // slots: the NULL registration and the evicted valued group.+ let (_, kind) = map.find_or_insert(200, a.clone(), b_idx, |v| *v == a);+ assert_eq!(kind, InsertKind::ReplacedNull);++ // Both freed slots must remain reusable. Otherwise repeated+ // conversions make the backing store grow without bound.+ assert!(map.insert_null(300, c.clone(), |v| *v == c));+ assert_eq!(map.store.len(), 2);++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs b/datafusion/physical-plan/src/aggregates/topk/priority_map.rsindex c74b648d37..f46cb22a7a 100644--- a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs+++ b/datafusion/physical-plan/src/aggregates/topk/priority_map.rs@@ -17,9 +17,10 @@//! A `Map<K, V>` / `PriorityQueue` combo that evicts the worst values after reaching `capacity`-use crate::aggregates::topk::hash_table::{ArrowHashTable, new_hash_table};+use crate::aggregates::topk::hash_table::{ArrowHashTable, InsertKind, new_hash_table};use crate::aggregates::topk::heap::{ArrowHeap, new_heap};-use arrow::array::ArrayRef;+use arrow::array::{ArrayRef, new_null_array};+use arrow::compute::concat;use arrow::datatypes::DataType;use datafusion_common::Result;@@ -29,6 +30,11 @@ pub struct PriorityMap {heap: Box<dyn ArrowHeap + Send>,capacity: usize,mapper: Vec<(usize, usize)>,+ val_type: DataType,+ /// Mirror of the map's all-NULL group count, kept as a plain field so the+ /// per-row `insert` path can check it without a `dyn` call (measured to+ /// regress the topk_aggregate benchmarks when read through the trait)+ null_count: usize,}impl PriorityMap {@@ -40,9 +46,11 @@ impl PriorityMap {) -> Result<Self> {Ok(Self {map: new_hash_table(capacity, key_type)?,- heap: new_heap(capacity, descending, val_type)?,+ heap: new_heap(capacity, descending, val_type.clone())?,capacity,mapper: Vec::with_capacity(capacity),+ val_type,+ null_count: 0,})}@@ -53,19 +61,47 @@ impl PriorityMap {pub fn insert(&mut self, row_idx: usize) -> Result<()> {assert!(self.map.len() <= self.capacity, "Overflow");+ debug_assert_eq!(self.null_count, 0);// if we're full, and the new val is worse than all our values, just bailif self.heap.is_worse(row_idx) {return Ok(());}+ self.insert_eligible(row_idx)+ }++ /// Insert a value while all-NULL groups are being tracked. This is kept+ /// separate from [`Self::insert`] so the common no-NULL path does not pay+ /// for NULL bookkeeping on every row.+ pub fn insert_with_null_groups(&mut self, row_idx: usize) -> Result<()> {+ // valued groups are capped at `capacity`; up to `capacity` additional+ // all-NULL groups may be tracked alongside them+ assert!(self.map.len() <= 2 * self.capacity, "Overflow");++ if self.heap.is_worse(row_idx) {+ // A group that was registered as all-NULL now has a value that+ // loses to the current top-k: it can no longer reach the top-k,+ // but it must not be emitted with a NULL value either+ if self.null_count > 0 && self.map.remove_if_null(row_idx) {+ self.null_count -= 1;+ }+ return Ok(());+ }+ self.insert_eligible(row_idx)+ }++ fn insert_eligible(&mut self, row_idx: usize) -> Result<()> {let map = &mut self.mapper;// handle new groups we haven't seen yetmap.clear();let replace_idx = self.heap.worst_map_idx();- let (map_idx, did_insert) = self.map.find_or_insert(row_idx, replace_idx);- if did_insert {+ let (map_idx, kind) = self.map.find_or_insert(row_idx, replace_idx);+ if kind == InsertKind::ReplacedNull {+ self.null_count -= 1;+ }+ if kind != InsertKind::Existing {self.heap.insert(row_idx, map_idx, map);self.map.update_heap_idx(map);return Ok(());@@ -80,9 +116,35 @@ impl PriorityMap {Ok(())}+ pub fn has_null_groups(&self) -> bool {+ self.null_count > 0+ }++ /// Track a group whose aggregate values are all NULL, so it can be emitted+ /// with a NULL value. MIN/MAX ignore NULL inputs, but an all-NULL group+ /// must still appear in the aggregation output; such groups all tie on the+ /// sort key, so tracking up to `capacity` of them preserves top-k semantics.+ pub fn insert_null(&mut self, row_idx: usize) {+ assert!(self.map.len() <= 2 * self.capacity, "Overflow");+ if self.map.insert_null(row_idx) {+ self.null_count += 1;+ }+ }+pub fn emit(&mut self) -> Result<Vec<ArrayRef>> {- let (vals, map_idxs) = self.heap.drain();+ let (vals, mut map_idxs) = self.heap.drain();+ // Groups whose values are all NULL are tracked in the map only;+ // append them with a NULL value so they are not lost from the output+ let null_idxs = self.map.null_map_idxs();+ let vals = if null_idxs.is_empty() {+ vals+ } else {+ map_idxs.extend(null_idxs.iter().copied());+ let nulls = new_null_array(&self.val_type, null_idxs.len());+ concat(&[vals.as_ref(), nulls.as_ref()])?+ };let ids = self.map.take_all(map_idxs);+ self.null_count = 0;Ok(vec![ids, vals])}@@ -495,6 +557,224 @@ mod tests {Ok(())}+ #[test]+ fn should_emit_all_null_groups() -> Result<()> {+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1", "2"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None, None]));+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, true)?;+ agg.set_batch(ids, vals);+ agg.insert_null(0);+ agg.insert_null(1);+ // re-registering an existing NULL group is a no-op+ agg.insert_null(0);++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 1 | |+ | 2 | |+ +----------+--------------++ "+ );++ Ok(())+ }++ #[test]+ fn should_emit_null_groups_alongside_valued_groups() -> Result<()> {+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1", "2", "3"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![Some(7), None, Some(3)]));+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 3, true)?;+ agg.set_batch(ids, vals);+ agg.insert(0)?;+ agg.insert_null(1);+ agg.insert_with_null_groups(2)?;++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 1 | 7 |+ | 3 | 3 |+ | 2 | |+ +----------+--------------++ "+ );++ Ok(())+ }++ #[test]+ fn should_cap_null_groups_at_limit() -> Result<()> {+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1", "2", "3", "4", "5"]));+ let vals: ArrayRef =+ Arc::new(Int64Array::from(vec![None, None, None, None, None]));+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, false)?;+ agg.set_batch(ids, vals);+ for row_idx in 0..5 {+ agg.insert_null(row_idx);+ }++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 1 | |+ | 2 | |+ +----------+--------------++ "+ );++ Ok(())+ }++ #[test]+ fn should_convert_null_group_to_valued() -> Result<()> {+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, true)?;++ // group "1" only produces NULLs in the first batch+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));+ agg.set_batch(ids, vals);+ agg.insert_null(0);++ // group "1" produces a value in a later batch+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![5]));+ agg.set_batch(ids, vals);+ agg.insert_with_null_groups(0)?;++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 1 | 5 |+ +----------+--------------++ "+ );++ Ok(())+ }++ #[test]+ fn should_not_duplicate_valued_group_as_null() -> Result<()> {+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, false)?;++ // group "1" produces a value in the first batch+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![5]));+ agg.set_batch(ids, vals);+ agg.insert(0)?;++ // group "1" only produces NULLs in a later batch+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));+ agg.set_batch(ids, vals);+ agg.insert_null(0);++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 1 | 5 |+ +----------+--------------++ "+ );++ Ok(())+ }++ #[test]+ fn should_evict_worst_when_converting_null_group() -> Result<()> {+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 1, true)?;++ // group "2" holds the single top-k slot+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["2"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![10]));+ agg.set_batch(ids, vals);+ agg.insert(0)?;++ // group "1" starts out all-NULL+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));+ agg.set_batch(ids, vals);+ agg.insert_null(0);++ // group "1" produces a better value and evicts group "2"+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![20]));+ agg.set_batch(ids, vals);+ agg.insert_with_null_groups(0)?;++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 1 | 20 |+ +----------+--------------++ "+ );++ Ok(())+ }++ #[test]+ fn should_drop_null_group_that_loses_to_topk() -> Result<()> {+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 1, true)?;++ // group "1" starts out all-NULL+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));+ agg.set_batch(ids, vals);+ agg.insert_null(0);++ // group "2" fills the single top-k slot+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["2"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![10]));+ agg.set_batch(ids, vals);+ agg.insert_with_null_groups(0)?;++ // group "1" produces a value that loses to the current top-k: the+ // group can no longer reach the top-k and must not be emitted as NULL+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![5]));+ agg.set_batch(ids, vals);+ agg.insert_with_null_groups(0)?;++ let cols = agg.emit()?;+ let batch = RecordBatch::try_new(test_schema(), cols)?;+ let actual = format!("{}", pretty_format_batches(&[batch])?);+ assert_snapshot!(actual, @r"+ +----------+--------------++ | trace_id | timestamp_ms |+ +----------+--------------++ | 2 | 10 |+ +----------+--------------++ "+ );++ Ok(())+ }+fn test_schema() -> SchemaRef {Arc::new(Schema::new(vec![Field::new("trace_id", DataType::Utf8, true),diff --git a/datafusion/sqllogictest/test_files/aggregates_topk.slt b/datafusion/sqllogictest/test_files/aggregates_topk.sltindex 39e3d91aa1..e2d453068a 100644--- a/datafusion/sqllogictest/test_files/aggregates_topk.slt+++ b/datafusion/sqllogictest/test_files/aggregates_topk.slt@@ -98,15 +98,15 @@ c 4a 1query TT-explain select trace_id, MAX(timestamp) from traces group by trace_id order by MAX(timestamp) desc limit 4;+explain select trace_id, MAX(timestamp) from traces group by trace_id order by MAX(timestamp) desc nulls last limit 4;----logical_plan-01)Sort: max(traces.timestamp) DESC NULLS FIRST, fetch=4+01)Sort: max(traces.timestamp) DESC NULLS LAST, fetch=402)--Aggregate: groupBy=[[traces.trace_id]], aggr=[[max(traces.timestamp)]]03)----TableScan: traces projection=[trace_id, timestamp]physical_plan-01)SortPreservingMergeExec: [max(traces.timestamp)@1 DESC], fetch=4-02)--SortExec: TopK(fetch=4), expr=[max(traces.timestamp)@1 DESC], preserve_partitioning=[true]+01)SortPreservingMergeExec: [max(traces.timestamp)@1 DESC NULLS LAST], fetch=4+02)--SortExec: TopK(fetch=4), expr=[max(traces.timestamp)@1 DESC NULLS LAST], preserve_partitioning=[true]03)----AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces.timestamp)], lim=[4]04)------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=105)--------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces.timestamp)], lim=[4]@@ -218,17 +218,17 @@ x zebraz mangoquery TT-explain select category, max(val) max_val from string_topk group by category order by max_val desc limit 2;+explain select category, max(val) max_val from string_topk group by category order by max_val desc nulls last limit 2;----logical_plan-01)Sort: max_val DESC NULLS FIRST, fetch=2+01)Sort: max_val DESC NULLS LAST, fetch=202)--Projection: string_topk.category, max(string_topk.val) AS max_val03)----Aggregate: groupBy=[[string_topk.category]], aggr=[[max(string_topk.val)]]04)------TableScan: string_topk projection=[category, val]physical_plan-01)SortPreservingMergeExec: [max_val@1 DESC], fetch=2+01)SortPreservingMergeExec: [max_val@1 DESC NULLS LAST], fetch=202)--ProjectionExec: expr=[category@0 as category, max(string_topk.val)@1 as max_val]-03)----SortExec: TopK(fetch=2), expr=[max(string_topk.val)@1 DESC], preserve_partitioning=[true]+03)----SortExec: TopK(fetch=2), expr=[max(string_topk.val)@1 DESC NULLS LAST], preserve_partitioning=[true]04)------AggregateExec: mode=FinalPartitioned, gby=[category@0 as category], aggr=[max(string_topk.val)], lim=[2]05)--------RepartitionExec: partitioning=Hash([category@0], 4), input_partitions=106)----------AggregateExec: mode=Partial, gby=[category@0 as category], aggr=[max(string_topk.val)], lim=[2]@@ -241,19 +241,19 @@ x zebraz mangoquery TT-explain select category, max(val) max_val from string_topk_view group by category order by max_val desc limit 2;+explain select category, max(val) max_val from string_topk_view group by category order by max_val desc nulls last limit 2;----logical_plan-01)Sort: max_val DESC NULLS FIRST, fetch=2+01)Sort: max_val DESC NULLS LAST, fetch=202)--Projection: string_topk_view.category, max(string_topk_view.val) AS max_val03)----Aggregate: groupBy=[[string_topk_view.category]], aggr=[[max(string_topk_view.val)]]04)------SubqueryAlias: string_topk_view05)--------Projection: string_topk.category AS category, string_topk.val AS val06)----------TableScan: string_topk projection=[category, val]physical_plan-01)SortPreservingMergeExec: [max_val@1 DESC], fetch=2+01)SortPreservingMergeExec: [max_val@1 DESC NULLS LAST], fetch=202)--ProjectionExec: expr=[category@0 as category, max(string_topk_view.val)@1 as max_val]-03)----SortExec: TopK(fetch=2), expr=[max(string_topk_view.val)@1 DESC], preserve_partitioning=[true]+03)----SortExec: TopK(fetch=2), expr=[max(string_topk_view.val)@1 DESC NULLS LAST], preserve_partitioning=[true]04)------AggregateExec: mode=FinalPartitioned, gby=[category@0 as category], aggr=[max(string_topk_view.val)], lim=[2]05)--------RepartitionExec: partitioning=Hash([category@0], 4), input_partitions=106)----------AggregateExec: mode=Partial, gby=[category@0 as category], aggr=[max(string_topk_view.val)], lim=[2]@@ -268,11 +268,13 @@ NULL 0 0c 1 2# Regression tests for string max with ORDER BY ... LIMIT to ensure schema stability+# Note: the NULL group has an all-NULL trace_id, so its max is NULL and ranks+# first under DESC NULLS FIRST (previously the group was dropped: issue #23440)query TTselect trace_id, max(trace_id) as max_trace from traces group by trace_id order by max_trace desc limit 2;----+NULL NULLc c-b bquery TTexplain select trace_id, max(trace_id) as max_trace from traces group by trace_id order by max_trace desc limit 2;@@ -286,9 +288,9 @@ physical_plan01)SortPreservingMergeExec: [max_trace@1 DESC], fetch=202)--ProjectionExec: expr=[trace_id@0 as trace_id, max(traces.trace_id)@1 as max_trace]03)----SortExec: TopK(fetch=2), expr=[max(traces.trace_id)@1 DESC], preserve_partitioning=[true]-04)------AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)], lim=[2]+04)------AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)]05)--------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=1-06)----------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)], lim=[2]+06)----------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)]07)------------DataSourceExec: partitions=1, partition_sizes=[1]@@ -303,15 +305,15 @@ AS SELECTFROM traces;query TT-explain select trace_id, MAX(timestamp) from traces_utf8view group by trace_id order by MAX(timestamp) desc limit 4;+explain select trace_id, MAX(timestamp) from traces_utf8view group by trace_id order by MAX(timestamp) desc nulls last limit 4;----logical_plan-01)Sort: max(traces_utf8view.timestamp) DESC NULLS FIRST, fetch=4+01)Sort: max(traces_utf8view.timestamp) DESC NULLS LAST, fetch=402)--Aggregate: groupBy=[[traces_utf8view.trace_id]], aggr=[[max(traces_utf8view.timestamp)]]03)----TableScan: traces_utf8view projection=[trace_id, timestamp]physical_plan-01)SortPreservingMergeExec: [max(traces_utf8view.timestamp)@1 DESC], fetch=4-02)--SortExec: TopK(fetch=4), expr=[max(traces_utf8view.timestamp)@1 DESC], preserve_partitioning=[true]+01)SortPreservingMergeExec: [max(traces_utf8view.timestamp)@1 DESC NULLS LAST], fetch=4+02)--SortExec: TopK(fetch=4), expr=[max(traces_utf8view.timestamp)@1 DESC NULLS LAST], preserve_partitioning=[true]03)----AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces_utf8view.timestamp)], lim=[4]04)------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=105)--------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces_utf8view.timestamp)], lim=[4]@@ -329,15 +331,15 @@ AS SELECTFROM traces;query TT-explain select trace_id, MAX(timestamp) from traces_largeutf8 group by trace_id order by MAX(timestamp) desc limit 4;+explain select trace_id, MAX(timestamp) from traces_largeutf8 group by trace_id order by MAX(timestamp) desc nulls last limit 4;----logical_plan-01)Sort: max(traces_largeutf8.timestamp) DESC NULLS FIRST, fetch=4+01)Sort: max(traces_largeutf8.timestamp) DESC NULLS LAST, fetch=402)--Aggregate: groupBy=[[traces_largeutf8.trace_id]], aggr=[[max(traces_largeutf8.timestamp)]]03)----TableScan: traces_largeutf8 projection=[trace_id, timestamp]physical_plan-01)SortPreservingMergeExec: [max(traces_largeutf8.timestamp)@1 DESC], fetch=4-02)--SortExec: TopK(fetch=4), expr=[max(traces_largeutf8.timestamp)@1 DESC], preserve_partitioning=[true]+01)SortPreservingMergeExec: [max(traces_largeutf8.timestamp)@1 DESC NULLS LAST], fetch=4+02)--SortExec: TopK(fetch=4), expr=[max(traces_largeutf8.timestamp)@1 DESC NULLS LAST], preserve_partitioning=[true]03)----AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces_largeutf8.timestamp)], lim=[4]04)------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=105)--------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces_largeutf8.timestamp)], lim=[4]@@ -585,3 +587,205 @@ drop table ids;statement okdrop table traces;++#######+# Regression tests for all-NULL groups in TopK aggregation (issues #23440, #22190):+# a group whose aggregate inputs are all NULL must be emitted with a NULL+# aggregate value instead of disappearing from the result+#######+statement ok+CREATE TABLE t0 AS SELECT * FROM (VALUES ('gamma', CAST(NULL AS DOUBLE))) v(s, y);++# MIN/MAX with NULLS FIRST must use regular aggregation because a group's+# aggregate can transition from NULL to non-NULL and worsen its rank.+query TT+explain select s, max(y) as max_y from t0 group by s order by max_y desc nulls first limit 3;+----+logical_plan+01)Sort: max_y DESC NULLS FIRST, fetch=3+02)--Projection: t0.s, max(t0.y) AS max_y+03)----Aggregate: groupBy=[[t0.s]], aggr=[[max(t0.y)]]+04)------TableScan: t0 projection=[s, y]+physical_plan+01)ProjectionExec: expr=[s@0 as s, max(t0.y)@1 as max_y]+02)--SortExec: TopK(fetch=3), expr=[max(t0.y)@1 DESC], preserve_partitioning=[false]+03)----AggregateExec: mode=SinglePartitioned, gby=[s@0 as s], aggr=[max(t0.y)]+04)------DataSourceExec: partitions=1, partition_sizes=[1]++# issue #23440: single all-NULL group, MAX DESC NULLS FIRST LIMIT 3+query R+SELECT max_y FROM (SELECT s, MAX(y) AS max_y FROM t0 GROUP BY s) ORDER BY max_y DESC NULLS FIRST LIMIT 3;+----+NULL++# issue #22190: single all-NULL group, MIN ASC NULLS LAST LIMIT 20+query TT+EXPLAIN SELECT min_y FROM (SELECT s, MIN(y) AS min_y FROM t0 GROUP BY s) ORDER BY min_y ASC NULLS LAST LIMIT 20;+----+logical_plan+01)Sort: min_y ASC NULLS LAST, fetch=20+02)--Projection: min(t0.y) AS min_y+03)----Aggregate: groupBy=[[t0.s]], aggr=[[min(t0.y)]]+04)------TableScan: t0 projection=[s, y]+physical_plan+01)SortExec: TopK(fetch=20), expr=[min_y@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--ProjectionExec: expr=[min(t0.y)@1 as min_y]+03)----AggregateExec: mode=SinglePartitioned, gby=[s@0 as s], aggr=[min(t0.y)], lim=[20]+04)------DataSourceExec: partitions=1, partition_sizes=[1]++query R+SELECT min_y FROM (SELECT s, MIN(y) AS min_y FROM t0 GROUP BY s) ORDER BY min_y ASC NULLS LAST LIMIT 20;+----+NULL++# one all-NULL group and one valued group, limit larger than the group count:+# both rows must be present+statement ok+CREATE TABLE topk_two_groups(s varchar, y bigint) AS VALUES+('a', CAST(NULL AS BIGINT)),+('b', 10),+('b', 20);++query TI+select s, max_y from (select s, max(y) as max_y from topk_two_groups group by s) order by max_y desc nulls first limit 10;+----+a NULL+b 20++# 5 all-NULL groups with LIMIT 2: exactly 2 rows survive+statement ok+CREATE TABLE topk_five_nulls(s varchar, y bigint) AS VALUES+('g1', CAST(NULL AS BIGINT)),+('g2', CAST(NULL AS BIGINT)),+('g3', CAST(NULL AS BIGINT)),+('g4', CAST(NULL AS BIGINT)),+('g5', CAST(NULL AS BIGINT));++query I+select max_y from (select s, max(y) as max_y from topk_five_nulls group by s) order by max_y desc nulls first limit 2;+----+NULL+NULL++# 2 all-NULL groups and 3 valued groups with LIMIT 4+statement ok+CREATE TABLE topk_mixed(s varchar, y bigint) AS VALUES+('n1', CAST(NULL AS BIGINT)),+('n2', CAST(NULL AS BIGINT)),+('v1', 10),+('v2', 20),+('v3', 30);++# DESC NULLS FIRST: both NULL groups rank before all values+query I+select max_y from (select s, max(y) as max_y from topk_mixed group by s) order by max_y desc nulls first limit 4;+----+NULL+NULL+30+20++# DESC NULLS LAST: NULL groups rank after all values+query I+select max_y from (select s, max(y) as max_y from topk_mixed group by s) order by max_y desc nulls last limit 4;+----+30+20+10+NULL++# an all-NULL group that later produces a value losing to the current top-k+# must not be emitted with a NULL value+statement ok+CREATE TABLE topk_null_then_value(s varchar, y bigint) AS VALUES+('g1', CAST(NULL AS BIGINT)),+('g2', 10),+('g1', 5);++query I+select max_y from (select s, max(y) as max_y from topk_null_then_value group by s) order by max_y desc nulls first limit 1;+----+10++# single-row batches force NULL and non-NULL rows of the same group into+# different batches: a -> 7, b -> 5, c -> NULL+statement ok+set datafusion.execution.batch_size = 1;++statement ok+CREATE TABLE topk_batches(s varchar, y bigint) AS VALUES+('a', CAST(NULL AS BIGINT)),+('b', 5),+('a', 3),+('c', CAST(NULL AS BIGINT)),+('b', CAST(NULL AS BIGINT)),+('a', 7);++query TI+select s, max_y from (select s, max(y) as max_y from topk_batches group by s) order by max_y desc nulls first limit 3;+----+c NULL+a 7+b 5++# NULLS FIRST is not monotonic for MIN/MAX aggregation: a group initially+# registered as NULL can later become valued, so a bounded TopK cannot safely+# discard other NULL candidates. This must fall back to regular aggregation.+statement ok+set datafusion.execution.target_partitions = 1;++statement ok+CREATE TABLE topk_null_backfill(s varchar, y bigint) AS VALUES+('a', CAST(NULL AS BIGINT)),+('b', CAST(NULL AS BIGINT)),+('c', CAST(NULL AS BIGINT)),+('a', 5);++query I+select max_y from (select s, max(y) as max_y from topk_null_backfill group by s) order by max_y desc nulls first limit 2;+----+NULL+NULL++# An evicted valued group must not be re-registered and emitted as all-NULL.+statement ok+CREATE TABLE topk_evicted_then_null(s varchar, y bigint) AS VALUES+('a', 10),+('b', 20),+('a', CAST(NULL AS BIGINT)),+('c', CAST(NULL AS BIGINT));++query TI+select s, max_y from (select s, max(y) as max_y from topk_evicted_then_null group by s) order by max_y desc nulls first limit 1;+----+c NULL++statement ok+set datafusion.execution.batch_size = 8192;++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+drop table topk_batches;++statement ok+drop table topk_evicted_then_null;++statement ok+drop table topk_null_backfill;++statement ok+drop table topk_null_then_value;++statement ok+drop table topk_mixed;++statement ok+drop table topk_five_nulls;++statement ok+drop table topk_two_groups;++statement ok+drop table t0;diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.sltindex de493d6e4a..c4f2bcf2fa 100644--- a/datafusion/sqllogictest/test_files/group_by.slt+++ b/datafusion/sqllogictest/test_files/group_by.slt@@ -4608,9 +4608,9 @@ logical_planphysical_plan01)SortPreservingMergeExec: [max(timestamp_table.t1)@1 DESC], fetch=402)--SortExec: TopK(fetch=4), expr=[max(timestamp_table.t1)@1 DESC], preserve_partitioning=[true]-03)----AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[max(timestamp_table.t1)], lim=[4]+03)----AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[max(timestamp_table.t1)]04)------RepartitionExec: partitioning=Hash([c2@0], 8), input_partitions=8-05)--------AggregateExec: mode=Partial, gby=[c2@1 as c2], aggr=[max(timestamp_table.t1)], lim=[4]+05)--------AggregateExec: mode=Partial, gby=[c2@1 as c2], aggr=[max(timestamp_table.t1)]06)----------RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=407)------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/3.csv]]}, projection=[t1, c2], file_type=csv, has_header=true