Skip to content

Apache DataFusion SQL Query Engine

fix: TopK aggregation drops groups whose MIN/MAX value is NULL (#23684)

kidf8b9ed81f0dc
diff --git a/datafusion/physical-optimizer/src/topk_aggregation.rs b/datafusion/physical-optimizer/src/topk_aggregation.rs
index e1779c04a6..0eddb5d550 100644
--- a/datafusion/physical-optimizer/src/topk_aggregation.rs
+++ b/datafusion/physical-optimizer/src/topk_aggregation.rs
@@ -46,6 +46,7 @@ impl TopKAggregation {
aggr: &AggregateExec,
order_by: &str,
order_desc: bool,
+ nulls_first: bool,
limit: usize,
) -> Option<Arc<dyn ExecutionPlan>> {
// Current only support single group key
@@ -66,6 +67,26 @@ impl TopKAggregation {
// Check if this is ordering by an aggregate function (MIN/MAX)
if let Some((field, desc)) = aggr.get_minmax_desc() {
+ // A nullable MIN/MAX starts as NULL and becomes non-NULL when the
+ // group sees its first value. With NULLS FIRST that transition
+ // worsens the group's rank, so a bounded aggregation cannot safely
+ // discard other NULL groups. Use regular aggregation for exact
+ // results. Non-nullable inputs never take this transition and can
+ // still use TopK.
+ let input_nullable = aggr
+ .aggr_expr()
+ .iter()
+ .exactly_one()
+ .ok()?
+ .expressions()
+ .into_iter()
+ .exactly_one()
+ .ok()?
+ .nullable(aggr.input_schema.as_ref())
+ .ok()?;
+ if nulls_first && input_nullable {
+ return None;
+ }
// ensure the sort direction matches aggregate function
if desc != order_desc {
return None;
@@ -100,6 +121,7 @@ impl TopKAggregation {
let order = sort.properties().output_ordering()?;
let order = order.iter().exactly_one().ok()?;
let order_desc = order.options.descending;
+ let nulls_first = order.options.nulls_first;
let order = order.expr.downcast_ref::<Column>()?;
let mut cur_col_name = order.name().to_string();
let limit = sort.fetch()?;
@@ -111,7 +133,13 @@ impl TopKAggregation {
}
if let Some(aggr) = plan.downcast_ref::<AggregateExec>() {
// either we run into an Aggregate and transform it
- match Self::transform_agg(aggr, &cur_col_name, order_desc, limit) {
+ match Self::transform_agg(
+ aggr,
+ &cur_col_name,
+ order_desc,
+ nulls_first,
+ limit,
+ ) {
None => cardinality_preserved = false,
Some(plan) => return Ok(Transformed::yes(plan)),
}
diff --git a/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rs
index 97f4662c11..193fdba4b0 100644
--- a/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/grouped_topk_stream.rs
@@ -149,11 +149,26 @@ impl GroupedTopKAggregateStream {
if has_nulls && self.is_group_by_only() {
self.null_group_seen = true;
}
+ // Keep the common no-NULL path free of NULL bookkeeping. Once a NULL
+ // group exists, use the NULL-aware path until it has been resolved.
+ let track_null_groups = !self.is_group_by_only()
+ && (has_nulls || self.priority_map.has_null_groups());
for row_idx in 0..len {
if has_nulls && vals.is_null(row_idx) {
+ // MIN/MAX ignore NULL inputs, but a group whose values are all
+ // NULL must still be emitted with a NULL aggregate value, so
+ // track it. (GROUP BY-only aggregations handle NULL group keys
+ // via `null_group_seen` instead.)
+ if !self.is_group_by_only() {
+ self.priority_map.insert_null(row_idx);
+ }
continue;
}
- self.priority_map.insert(row_idx)?;
+ if track_null_groups {
+ self.priority_map.insert_with_null_groups(row_idx)?;
+ } else {
+ self.priority_map.insert(row_idx)?;
+ }
}
Ok(())
}
diff --git a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs
index 694780f085..adc8f8c315 100644
--- a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs
+++ b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs
@@ -39,6 +39,11 @@ pub trait KeyType: Clone + Comparable + Debug {}
impl<T> KeyType for T where T: Clone + Comparable + Debug {}
+/// `heap_idx` assigned to groups whose aggregate values are all NULL. Such
+/// groups are tracked in the hash table only (they never enter the heap), so
+/// they can be emitted with a NULL aggregate value at the end.
+const NULL_HEAP_IDX: usize = usize::MAX;
+
/// An entry in our hash table that:
/// 1. memoizes the hash
/// 2. contains the key (ID)
@@ -57,10 +62,25 @@ struct TopKHashTable<ID: KeyType> {
map: HashTable<usize>,
// Store the actual items separately to allow for index-based access
store: Vec<Option<HashTableItem<ID>>>,
- // Free index in the store for reuse
- free_index: Option<usize>,
+ // Free indexes in the store for reuse
+ free_indices: Vec<usize>,
// The maximum number of entries allowed
limit: usize,
+ // Number of entries registered as all-NULL (heap_idx == NULL_HEAP_IDX)
+ null_count: usize,
+}
+
+/// Outcome of [`ArrowHashTable::find_or_insert`], letting the caller keep its
+/// own all-NULL group accounting in sync without an extra lookup.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum InsertKind {
+ /// The group already existed as a valued group
+ Existing,
+ /// The group was newly inserted as a valued group
+ New,
+ /// The group was registered as all-NULL and has now been converted into a
+ /// valued group
+ ReplacedNull,
}
/// An interface to hide the generic type signature of TopKHashTable behind arrow arrays
@@ -70,7 +90,20 @@ pub trait ArrowHashTable {
fn update_heap_idx(&mut self, mapper: &[(usize, usize)]);
fn heap_idx_at(&self, map_idx: usize) -> usize;
fn take_all(&mut self, indexes: Vec<usize>) -> ArrayRef;
- fn find_or_insert(&mut self, row_idx: usize, replace_idx: usize) -> (usize, bool);
+ fn find_or_insert(
+ &mut self,
+ row_idx: usize,
+ replace_idx: usize,
+ ) -> (usize, InsertKind);
+ /// Register the group at `row_idx` as all-NULL. Returns true if it was
+ /// newly registered; false if the group is already tracked or the NULL
+ /// group limit has been reached.
+ fn insert_null(&mut self, row_idx: usize) -> bool;
+ /// Remove the group at `row_idx` if it is registered as all-NULL. Returns
+ /// true if a NULL registration was removed.
+ fn remove_if_null(&mut self, row_idx: usize) -> bool;
+ /// Store indexes of all groups registered as all-NULL
+ fn null_map_idxs(&self) -> Vec<usize>;
}
/// Returns true if the given data type can be used as a top-K aggregation hash key.
@@ -150,6 +183,13 @@ impl StringHashTable {
Some(value.to_string())
}
}
+
+ /// Computes the id and its hash for the given row, for hash table lookups
+ fn id_and_hash(&self, row_idx: usize) -> (Option<String>, u64) {
+ let id = self.extract_string_value(row_idx);
+ let hash = self.rnd.hash_one(id.as_deref());
+ (id, hash)
+ }
}
impl ArrowHashTable for StringHashTable {
@@ -179,7 +219,11 @@ impl ArrowHashTable for StringHashTable {
}
}
- fn find_or_insert(&mut self, row_idx: usize, replace_idx: usize) -> (usize, bool) {
+ fn find_or_insert(
+ &mut self,
+ row_idx: usize,
+ replace_idx: usize,
+ ) -> (usize, InsertKind) {
let id = self.extract_string_value(row_idx);
// Compute hash and create equality closure for hash table lookup.
@@ -190,6 +234,23 @@ impl ArrowHashTable for StringHashTable {
// Use entry API to avoid double lookup
self.map.find_or_insert(hash, id, replace_idx, eq)
}
+
+ fn insert_null(&mut self, row_idx: usize) -> bool {
+ let (id, hash) = self.id_and_hash(row_idx);
+ let id_for_eq = id.clone();
+ let eq = move |mi: &Option<String>| id_for_eq.as_deref() == mi.as_deref();
+ self.map.insert_null(hash, id, eq)
+ }
+
+ fn remove_if_null(&mut self, row_idx: usize) -> bool {
+ let (id, hash) = self.id_and_hash(row_idx);
+ let eq = move |mi: &Option<String>| id.as_deref() == mi.as_deref();
+ self.map.remove_if_null(hash, eq)
+ }
+
+ fn null_map_idxs(&self) -> Vec<usize> {
+ self.map.null_map_idxs()
+ }
}
impl<VAL: ArrowPrimitiveType> PrimitiveHashTable<VAL>
@@ -210,6 +271,18 @@ where
kt,
}
}
+
+ /// Computes the id and its hash for the given row, for hash table lookups
+ fn id_and_hash(&self, row_idx: usize) -> (Option<VAL::Native>, u64) {
+ let ids = self.owned.as_primitive::<VAL>();
+ let id: Option<VAL::Native> = if ids.is_null(row_idx) {
+ None
+ } else {
+ Some(ids.value(row_idx))
+ };
+ let hash: u64 = id.hash(&self.rnd);
+ (id, hash)
+ }
}
impl<VAL: ArrowPrimitiveType> ArrowHashTable for PrimitiveHashTable<VAL>
@@ -247,7 +320,11 @@ where
Arc::new(ids)
}
- fn find_or_insert(&mut self, row_idx: usize, replace_idx: usize) -> (usize, bool) {
+ fn find_or_insert(
+ &mut self,
+ row_idx: usize,
+ replace_idx: usize,
+ ) -> (usize, InsertKind) {
let ids = self.owned.as_primitive::<VAL>();
let id: Option<VAL::Native> = if ids.is_null(row_idx) {
None
@@ -261,6 +338,22 @@ where
// Use entry API to avoid double lookup
self.map.find_or_insert(hash, id, replace_idx, eq)
}
+
+ fn insert_null(&mut self, row_idx: usize) -> bool {
+ let (id, hash) = self.id_and_hash(row_idx);
+ let eq = move |mi: &Option<VAL::Native>| id == *mi;
+ self.map.insert_null(hash, id, eq)
+ }
+
+ fn remove_if_null(&mut self, row_idx: usize) -> bool {
+ let (id, hash) = self.id_and_hash(row_idx);
+ let eq = move |mi: &Option<VAL::Native>| id == *mi;
+ self.map.remove_if_null(hash, eq)
+ }
+
+ fn null_map_idxs(&self) -> Vec<usize> {
+ self.map.null_map_idxs()
+ }
}
use hashbrown::hash_table::Entry;
@@ -269,8 +362,9 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {
Self {
map: HashTable::with_capacity(capacity),
store: Vec::with_capacity(capacity),
- free_index: None,
+ free_indices: Vec::new(),
limit,
+ null_count: 0,
}
}
@@ -278,25 +372,33 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {
self.store[map_idx].as_ref().unwrap().heap_idx
}
- pub fn remove_if_full(&mut self, replace_idx: usize) -> usize {
- if self.map.len() >= self.limit {
- let item_to_remove = self.store[replace_idx].as_ref().unwrap();
- let hash = item_to_remove.hash;
- let id_to_remove = &item_to_remove.id;
-
- let eq = |&idx: &usize| self.store[idx].as_ref().unwrap().id == *id_to_remove;
- let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;
- match self.map.entry(hash, eq, hasher) {
- Entry::Occupied(entry) => {
- let (removed_idx, _) = entry.remove();
- self.store[removed_idx] = None;
- self.free_index = Some(removed_idx);
- }
- Entry::Vacant(_) => unreachable!(),
+ /// Remove the entry stored at `map_idx`, freeing its store slot for reuse
+ fn remove_at(&mut self, map_idx: usize) {
+ let item_to_remove = self.store[map_idx].as_ref().unwrap();
+ let hash = item_to_remove.hash;
+ let id_to_remove = &item_to_remove.id;
+
+ let eq = |&idx: &usize| self.store[idx].as_ref().unwrap().id == *id_to_remove;
+ let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;
+ match self.map.entry(hash, eq, hasher) {
+ Entry::Occupied(entry) => {
+ let (removed_idx, _) = entry.remove();
+ self.store[removed_idx] = None;
+ self.free_indices.push(removed_idx);
}
+ Entry::Vacant(_) => unreachable!(),
+ }
+ }
+
+ pub fn remove_if_full(&mut self, replace_idx: usize) -> usize {
+ // All-NULL groups are tracked outside the heap, so only valued
+ // groups count towards the limit here
+ let valued_len = self.map.len() - self.null_count;
+ if valued_len >= self.limit {
+ self.remove_at(replace_idx);
0 // if full, always replace top node
} else {
- self.map.len() // if we're not full, always append to end
+ valued_len // if we're not full, always append to end
}
}
@@ -307,7 +409,8 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {
}
/// Find an existing entry or insert a new one, avoiding double hash table lookup.
- /// Returns (map_idx, is_new) where is_new indicates if this was a new insertion.
+ /// Returns (map_idx, kind) where kind describes whether the group already
+ /// existed, was newly inserted, or was converted from an all-NULL group.
/// If inserting a new entry and the table is full, replaces the entry at replace_idx.
pub fn find_or_insert(
&mut self,
@@ -315,19 +418,28 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {
id: ID,
replace_idx: usize,
mut eq: impl FnMut(&ID) -> bool,
- ) -> (usize, bool) {
+ ) -> (usize, InsertKind) {
// Check if entry exists - this is the only hash table lookup
+ let mut replaced_null = false;
{
let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);
if let Some(&map_idx) = self.map.find(hash, eq_fn) {
- return (map_idx, false);
+ if self.store[map_idx].as_ref().unwrap().heap_idx == NULL_HEAP_IDX {
+ // This group was registered as all-NULL but now produced a
+ // value: unregister it so it is inserted as a valued group
+ self.remove_at(map_idx);
+ self.null_count -= 1;
+ replaced_null = true;
+ } else {
+ return (map_idx, InsertKind::Existing);
+ }
}
}
// Entry doesn't exist - compute heap_idx and prepare item
let heap_idx = self.remove_if_full(replace_idx);
let mi = HashTableItem::new(hash, id, heap_idx);
- let store_idx = if let Some(idx) = self.free_index.take() {
+ let store_idx = if let Some(idx) = self.free_indices.pop() {
self.store[idx] = Some(mi);
idx
} else {
@@ -343,7 +455,80 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {
// Insert without checking again since we already confirmed it doesn't exist
self.map.insert_unique(hash, store_idx, hasher);
- (store_idx, true)
+ let kind = if replaced_null {
+ InsertKind::ReplacedNull
+ } else {
+ InsertKind::New
+ };
+ (store_idx, kind)
+ }
+
+ /// Register a group whose aggregate values are all NULL, unless it is
+ /// already tracked. NULL groups are stored with a sentinel `heap_idx` and
+ /// never enter the heap. At most `limit` NULL groups are tracked: they all
+ /// tie on the sort key, so any `limit` of them is a valid top-k superset.
+ /// Returns true if the group was newly registered.
+ pub fn insert_null(
+ &mut self,
+ hash: u64,
+ id: ID,
+ mut eq: impl FnMut(&ID) -> bool,
+ ) -> bool {
+ {
+ let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);
+ if self.map.find(hash, eq_fn).is_some() {
+ return false;
+ }
+ }
+ if self.null_count >= self.limit {
+ return false;
+ }
+
+ let mi = HashTableItem::new(hash, id, NULL_HEAP_IDX);
+ let store_idx = if let Some(idx) = self.free_indices.pop() {
+ self.store[idx] = Some(mi);
+ idx
+ } else {
+ self.store.push(Some(mi));
+ self.store.len() - 1
+ };
+
+ let hasher = |idx: &usize| self.store[*idx].as_ref().unwrap().hash;
+ if self.map.len() == self.map.capacity() {
+ self.map.reserve(self.limit, hasher);
+ }
+ self.map.insert_unique(hash, store_idx, hasher);
+ self.null_count += 1;
+ true
+ }
+
+ /// Remove the given group if it is registered as all-NULL. Used when an
+ /// all-NULL group produces a value that loses to the current top-k: the
+ /// group can no longer reach the top-k, but it must not be emitted with a
+ /// NULL value either. Returns true if a NULL registration was removed.
+ pub fn remove_if_null(&mut self, hash: u64, mut eq: impl FnMut(&ID) -> bool) -> bool {
+ let eq_fn = |idx: &usize| eq(&self.store[*idx].as_ref().unwrap().id);
+ if let Some(&map_idx) = self.map.find(hash, eq_fn)
+ && self.store[map_idx].as_ref().unwrap().heap_idx == NULL_HEAP_IDX
+ {
+ self.remove_at(map_idx);
+ self.null_count -= 1;
+ return true;
+ }
+ false
+ }
+
+ /// Store indexes of all groups registered as all-NULL
+ pub fn null_map_idxs(&self) -> Vec<usize> {
+ self.store
+ .iter()
+ .enumerate()
+ .filter_map(|(idx, item)| {
+ item.as_ref()
+ .filter(|item| item.heap_idx == NULL_HEAP_IDX)
+ .map(|_| idx)
+ })
+ .collect()
}
pub fn len(&self) -> usize {
@@ -357,7 +542,8 @@ impl<ID: KeyType + PartialEq> TopKHashTable<ID> {
.collect();
self.map.clear();
self.store.clear();
- self.free_index = None;
+ self.free_indices.clear();
+ self.null_count = 0;
ids
}
}
@@ -453,9 +639,9 @@ mod tests {
for (heap_idx, id) in ["1", "2", "3", "4", "5"].iter().enumerate() {
let value = Some(id.to_string());
let hash = heap_idx as u64;
- let (map_idx, is_new) =
+ let (map_idx, kind) =
map.find_or_insert(hash, value.clone(), heap_idx, |v| *v == value);
- assert!(is_new, "Entry should be new");
+ assert_eq!(kind, InsertKind::New, "Entry should be new");
heap_to_map.insert(heap_idx, map_idx);
}
@@ -477,4 +663,65 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn should_track_null_groups() -> Result<()> {
+ let mut map = TopKHashTable::<Option<String>>::new(2, 10);
+
+ let a = Some("a".to_string());
+ let b = Some("b".to_string());
+ let c = Some("c".to_string());
+
+ // register two all-NULL groups; the third exceeds the NULL group limit
+ assert!(map.insert_null(100, a.clone(), |v| *v == a));
+ assert!(map.insert_null(200, b.clone(), |v| *v == b));
+ assert!(!map.insert_null(300, c.clone(), |v| *v == c));
+ // re-registering an existing NULL group is a no-op
+ assert!(!map.insert_null(100, a.clone(), |v| *v == a));
+ assert_eq!(map.null_count, 2);
+ assert_eq!(map.null_map_idxs(), vec![0, 1]);
+
+ // a valued insert for a NULL group converts it to a valued group
+ let (map_idx, kind) = map.find_or_insert(200, b.clone(), 0, |v| *v == b);
+ assert_eq!(kind, InsertKind::ReplacedNull, "NULL group should convert");
+ assert_eq!(map.heap_idx_at(map_idx), 0, "Heap should append at 0");
+ assert_eq!(map.null_count, 1);
+ assert_eq!(map.null_map_idxs(), vec![0]);
+
+ // remove the remaining NULL group; removing twice is a no-op
+ map.remove_if_null(100, |v| *v == a);
+ assert_eq!(map.null_count, 0);
+ assert!(map.null_map_idxs().is_empty());
+ map.remove_if_null(100, |v| *v == a);
+ // removing a valued group via remove_if_null is a no-op
+ map.remove_if_null(200, |v| *v == b);
+ assert_eq!(map.len(), 1);
+
+ Ok(())
+ }
+
+ #[test]
+ fn should_reuse_all_freed_store_slots() -> Result<()> {
+ let mut map = TopKHashTable::<Option<String>>::new(1, 10);
+
+ let a = Some("a".to_string());
+ let b = Some("b".to_string());
+ let c = Some("c".to_string());
+
+ let (b_idx, kind) = map.find_or_insert(100, b.clone(), 0, |v| *v == b);
+ assert_eq!(kind, InsertKind::New);
+ assert!(map.insert_null(200, a.clone(), |v| *v == a));
+
+ // Converting a NULL group while the valued heap is full frees two
+ // slots: the NULL registration and the evicted valued group.
+ let (_, kind) = map.find_or_insert(200, a.clone(), b_idx, |v| *v == a);
+ assert_eq!(kind, InsertKind::ReplacedNull);
+
+ // Both freed slots must remain reusable. Otherwise repeated
+ // conversions make the backing store grow without bound.
+ assert!(map.insert_null(300, c.clone(), |v| *v == c));
+ assert_eq!(map.store.len(), 2);
+
+ Ok(())
+ }
}
diff --git a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs b/datafusion/physical-plan/src/aggregates/topk/priority_map.rs
index c74b648d37..f46cb22a7a 100644
--- a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs
+++ b/datafusion/physical-plan/src/aggregates/topk/priority_map.rs
@@ -17,9 +17,10 @@
//! A `Map<K, V>` / `PriorityQueue` combo that evicts the worst values after reaching `capacity`
-use crate::aggregates::topk::hash_table::{ArrowHashTable, new_hash_table};
+use crate::aggregates::topk::hash_table::{ArrowHashTable, InsertKind, new_hash_table};
use crate::aggregates::topk::heap::{ArrowHeap, new_heap};
-use arrow::array::ArrayRef;
+use arrow::array::{ArrayRef, new_null_array};
+use arrow::compute::concat;
use arrow::datatypes::DataType;
use datafusion_common::Result;
@@ -29,6 +30,11 @@ pub struct PriorityMap {
heap: Box<dyn ArrowHeap + Send>,
capacity: usize,
mapper: Vec<(usize, usize)>,
+ val_type: DataType,
+ /// Mirror of the map's all-NULL group count, kept as a plain field so the
+ /// per-row `insert` path can check it without a `dyn` call (measured to
+ /// regress the topk_aggregate benchmarks when read through the trait)
+ null_count: usize,
}
impl PriorityMap {
@@ -40,9 +46,11 @@ impl PriorityMap {
) -> Result<Self> {
Ok(Self {
map: new_hash_table(capacity, key_type)?,
- heap: new_heap(capacity, descending, val_type)?,
+ heap: new_heap(capacity, descending, val_type.clone())?,
capacity,
mapper: Vec::with_capacity(capacity),
+ val_type,
+ null_count: 0,
})
}
@@ -53,19 +61,47 @@ impl PriorityMap {
pub fn insert(&mut self, row_idx: usize) -> Result<()> {
assert!(self.map.len() <= self.capacity, "Overflow");
+ debug_assert_eq!(self.null_count, 0);
// if we're full, and the new val is worse than all our values, just bail
if self.heap.is_worse(row_idx) {
return Ok(());
}
+ self.insert_eligible(row_idx)
+ }
+
+ /// Insert a value while all-NULL groups are being tracked. This is kept
+ /// separate from [`Self::insert`] so the common no-NULL path does not pay
+ /// for NULL bookkeeping on every row.
+ pub fn insert_with_null_groups(&mut self, row_idx: usize) -> Result<()> {
+ // valued groups are capped at `capacity`; up to `capacity` additional
+ // all-NULL groups may be tracked alongside them
+ assert!(self.map.len() <= 2 * self.capacity, "Overflow");
+
+ if self.heap.is_worse(row_idx) {
+ // A group that was registered as all-NULL now has a value that
+ // loses to the current top-k: it can no longer reach the top-k,
+ // but it must not be emitted with a NULL value either
+ if self.null_count > 0 && self.map.remove_if_null(row_idx) {
+ self.null_count -= 1;
+ }
+ return Ok(());
+ }
+ self.insert_eligible(row_idx)
+ }
+
+ fn insert_eligible(&mut self, row_idx: usize) -> Result<()> {
let map = &mut self.mapper;
// handle new groups we haven't seen yet
map.clear();
let replace_idx = self.heap.worst_map_idx();
- let (map_idx, did_insert) = self.map.find_or_insert(row_idx, replace_idx);
- if did_insert {
+ let (map_idx, kind) = self.map.find_or_insert(row_idx, replace_idx);
+ if kind == InsertKind::ReplacedNull {
+ self.null_count -= 1;
+ }
+ if kind != InsertKind::Existing {
self.heap.insert(row_idx, map_idx, map);
self.map.update_heap_idx(map);
return Ok(());
@@ -80,9 +116,35 @@ impl PriorityMap {
Ok(())
}
+ pub fn has_null_groups(&self) -> bool {
+ self.null_count > 0
+ }
+
+ /// Track a group whose aggregate values are all NULL, so it can be emitted
+ /// with a NULL value. MIN/MAX ignore NULL inputs, but an all-NULL group
+ /// must still appear in the aggregation output; such groups all tie on the
+ /// sort key, so tracking up to `capacity` of them preserves top-k semantics.
+ pub fn insert_null(&mut self, row_idx: usize) {
+ assert!(self.map.len() <= 2 * self.capacity, "Overflow");
+ if self.map.insert_null(row_idx) {
+ self.null_count += 1;
+ }
+ }
+
pub fn emit(&mut self) -> Result<Vec<ArrayRef>> {
- let (vals, map_idxs) = self.heap.drain();
+ let (vals, mut map_idxs) = self.heap.drain();
+ // Groups whose values are all NULL are tracked in the map only;
+ // append them with a NULL value so they are not lost from the output
+ let null_idxs = self.map.null_map_idxs();
+ let vals = if null_idxs.is_empty() {
+ vals
+ } else {
+ map_idxs.extend(null_idxs.iter().copied());
+ let nulls = new_null_array(&self.val_type, null_idxs.len());
+ concat(&[vals.as_ref(), nulls.as_ref()])?
+ };
let ids = self.map.take_all(map_idxs);
+ self.null_count = 0;
Ok(vec![ids, vals])
}
@@ -495,6 +557,224 @@ mod tests {
Ok(())
}
+ #[test]
+ fn should_emit_all_null_groups() -> Result<()> {
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1", "2"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None, None]));
+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, true)?;
+ agg.set_batch(ids, vals);
+ agg.insert_null(0);
+ agg.insert_null(1);
+ // re-registering an existing NULL group is a no-op
+ agg.insert_null(0);
+
+ let cols = agg.emit()?;
+ let batch = RecordBatch::try_new(test_schema(), cols)?;
+ let actual = format!("{}", pretty_format_batches(&[batch])?);
+ assert_snapshot!(actual, @r"
+ +----------+--------------+
+ | trace_id | timestamp_ms |
+ +----------+--------------+
+ | 1 | |
+ | 2 | |
+ +----------+--------------+
+ "
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn should_emit_null_groups_alongside_valued_groups() -> Result<()> {
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1", "2", "3"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![Some(7), None, Some(3)]));
+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 3, true)?;
+ agg.set_batch(ids, vals);
+ agg.insert(0)?;
+ agg.insert_null(1);
+ agg.insert_with_null_groups(2)?;
+
+ let cols = agg.emit()?;
+ let batch = RecordBatch::try_new(test_schema(), cols)?;
+ let actual = format!("{}", pretty_format_batches(&[batch])?);
+ assert_snapshot!(actual, @r"
+ +----------+--------------+
+ | trace_id | timestamp_ms |
+ +----------+--------------+
+ | 1 | 7 |
+ | 3 | 3 |
+ | 2 | |
+ +----------+--------------+
+ "
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn should_cap_null_groups_at_limit() -> Result<()> {
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1", "2", "3", "4", "5"]));
+ let vals: ArrayRef =
+ Arc::new(Int64Array::from(vec![None, None, None, None, None]));
+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, false)?;
+ agg.set_batch(ids, vals);
+ for row_idx in 0..5 {
+ agg.insert_null(row_idx);
+ }
+
+ let cols = agg.emit()?;
+ let batch = RecordBatch::try_new(test_schema(), cols)?;
+ let actual = format!("{}", pretty_format_batches(&[batch])?);
+ assert_snapshot!(actual, @r"
+ +----------+--------------+
+ | trace_id | timestamp_ms |
+ +----------+--------------+
+ | 1 | |
+ | 2 | |
+ +----------+--------------+
+ "
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn should_convert_null_group_to_valued() -> Result<()> {
+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, true)?;
+
+ // group "1" only produces NULLs in the first batch
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));
+ agg.set_batch(ids, vals);
+ agg.insert_null(0);
+
+ // group "1" produces a value in a later batch
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![5]));
+ agg.set_batch(ids, vals);
+ agg.insert_with_null_groups(0)?;
+
+ let cols = agg.emit()?;
+ let batch = RecordBatch::try_new(test_schema(), cols)?;
+ let actual = format!("{}", pretty_format_batches(&[batch])?);
+ assert_snapshot!(actual, @r"
+ +----------+--------------+
+ | trace_id | timestamp_ms |
+ +----------+--------------+
+ | 1 | 5 |
+ +----------+--------------+
+ "
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn should_not_duplicate_valued_group_as_null() -> Result<()> {
+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 2, false)?;
+
+ // group "1" produces a value in the first batch
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![5]));
+ agg.set_batch(ids, vals);
+ agg.insert(0)?;
+
+ // group "1" only produces NULLs in a later batch
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));
+ agg.set_batch(ids, vals);
+ agg.insert_null(0);
+
+ let cols = agg.emit()?;
+ let batch = RecordBatch::try_new(test_schema(), cols)?;
+ let actual = format!("{}", pretty_format_batches(&[batch])?);
+ assert_snapshot!(actual, @r"
+ +----------+--------------+
+ | trace_id | timestamp_ms |
+ +----------+--------------+
+ | 1 | 5 |
+ +----------+--------------+
+ "
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn should_evict_worst_when_converting_null_group() -> Result<()> {
+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 1, true)?;
+
+ // group "2" holds the single top-k slot
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["2"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![10]));
+ agg.set_batch(ids, vals);
+ agg.insert(0)?;
+
+ // group "1" starts out all-NULL
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));
+ agg.set_batch(ids, vals);
+ agg.insert_null(0);
+
+ // group "1" produces a better value and evicts group "2"
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![20]));
+ agg.set_batch(ids, vals);
+ agg.insert_with_null_groups(0)?;
+
+ let cols = agg.emit()?;
+ let batch = RecordBatch::try_new(test_schema(), cols)?;
+ let actual = format!("{}", pretty_format_batches(&[batch])?);
+ assert_snapshot!(actual, @r"
+ +----------+--------------+
+ | trace_id | timestamp_ms |
+ +----------+--------------+
+ | 1 | 20 |
+ +----------+--------------+
+ "
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn should_drop_null_group_that_loses_to_topk() -> Result<()> {
+ let mut agg = PriorityMap::new(DataType::Utf8, DataType::Int64, 1, true)?;
+
+ // group "1" starts out all-NULL
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![None]));
+ agg.set_batch(ids, vals);
+ agg.insert_null(0);
+
+ // group "2" fills the single top-k slot
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["2"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![10]));
+ agg.set_batch(ids, vals);
+ agg.insert_with_null_groups(0)?;
+
+ // group "1" produces a value that loses to the current top-k: the
+ // group can no longer reach the top-k and must not be emitted as NULL
+ let ids: ArrayRef = Arc::new(StringArray::from(vec!["1"]));
+ let vals: ArrayRef = Arc::new(Int64Array::from(vec![5]));
+ agg.set_batch(ids, vals);
+ agg.insert_with_null_groups(0)?;
+
+ let cols = agg.emit()?;
+ let batch = RecordBatch::try_new(test_schema(), cols)?;
+ let actual = format!("{}", pretty_format_batches(&[batch])?);
+ assert_snapshot!(actual, @r"
+ +----------+--------------+
+ | trace_id | timestamp_ms |
+ +----------+--------------+
+ | 2 | 10 |
+ +----------+--------------+
+ "
+ );
+
+ Ok(())
+ }
+
fn test_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("trace_id", DataType::Utf8, true),
diff --git a/datafusion/sqllogictest/test_files/aggregates_topk.slt b/datafusion/sqllogictest/test_files/aggregates_topk.slt
index 39e3d91aa1..e2d453068a 100644
--- a/datafusion/sqllogictest/test_files/aggregates_topk.slt
+++ b/datafusion/sqllogictest/test_files/aggregates_topk.slt
@@ -98,15 +98,15 @@ c 4
a 1
query TT
-explain select trace_id, MAX(timestamp) from traces group by trace_id order by MAX(timestamp) desc limit 4;
+explain select trace_id, MAX(timestamp) from traces group by trace_id order by MAX(timestamp) desc nulls last limit 4;
----
logical_plan
-01)Sort: max(traces.timestamp) DESC NULLS FIRST, fetch=4
+01)Sort: max(traces.timestamp) DESC NULLS LAST, fetch=4
02)--Aggregate: groupBy=[[traces.trace_id]], aggr=[[max(traces.timestamp)]]
03)----TableScan: traces projection=[trace_id, timestamp]
physical_plan
-01)SortPreservingMergeExec: [max(traces.timestamp)@1 DESC], fetch=4
-02)--SortExec: TopK(fetch=4), expr=[max(traces.timestamp)@1 DESC], preserve_partitioning=[true]
+01)SortPreservingMergeExec: [max(traces.timestamp)@1 DESC NULLS LAST], fetch=4
+02)--SortExec: TopK(fetch=4), expr=[max(traces.timestamp)@1 DESC NULLS LAST], preserve_partitioning=[true]
03)----AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces.timestamp)], lim=[4]
04)------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=1
05)--------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces.timestamp)], lim=[4]
@@ -218,17 +218,17 @@ x zebra
z mango
query TT
-explain select category, max(val) max_val from string_topk group by category order by max_val desc limit 2;
+explain select category, max(val) max_val from string_topk group by category order by max_val desc nulls last limit 2;
----
logical_plan
-01)Sort: max_val DESC NULLS FIRST, fetch=2
+01)Sort: max_val DESC NULLS LAST, fetch=2
02)--Projection: string_topk.category, max(string_topk.val) AS max_val
03)----Aggregate: groupBy=[[string_topk.category]], aggr=[[max(string_topk.val)]]
04)------TableScan: string_topk projection=[category, val]
physical_plan
-01)SortPreservingMergeExec: [max_val@1 DESC], fetch=2
+01)SortPreservingMergeExec: [max_val@1 DESC NULLS LAST], fetch=2
02)--ProjectionExec: expr=[category@0 as category, max(string_topk.val)@1 as max_val]
-03)----SortExec: TopK(fetch=2), expr=[max(string_topk.val)@1 DESC], preserve_partitioning=[true]
+03)----SortExec: TopK(fetch=2), expr=[max(string_topk.val)@1 DESC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[category@0 as category], aggr=[max(string_topk.val)], lim=[2]
05)--------RepartitionExec: partitioning=Hash([category@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[category@0 as category], aggr=[max(string_topk.val)], lim=[2]
@@ -241,19 +241,19 @@ x zebra
z mango
query TT
-explain select category, max(val) max_val from string_topk_view group by category order by max_val desc limit 2;
+explain select category, max(val) max_val from string_topk_view group by category order by max_val desc nulls last limit 2;
----
logical_plan
-01)Sort: max_val DESC NULLS FIRST, fetch=2
+01)Sort: max_val DESC NULLS LAST, fetch=2
02)--Projection: string_topk_view.category, max(string_topk_view.val) AS max_val
03)----Aggregate: groupBy=[[string_topk_view.category]], aggr=[[max(string_topk_view.val)]]
04)------SubqueryAlias: string_topk_view
05)--------Projection: string_topk.category AS category, string_topk.val AS val
06)----------TableScan: string_topk projection=[category, val]
physical_plan
-01)SortPreservingMergeExec: [max_val@1 DESC], fetch=2
+01)SortPreservingMergeExec: [max_val@1 DESC NULLS LAST], fetch=2
02)--ProjectionExec: expr=[category@0 as category, max(string_topk_view.val)@1 as max_val]
-03)----SortExec: TopK(fetch=2), expr=[max(string_topk_view.val)@1 DESC], preserve_partitioning=[true]
+03)----SortExec: TopK(fetch=2), expr=[max(string_topk_view.val)@1 DESC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[category@0 as category], aggr=[max(string_topk_view.val)], lim=[2]
05)--------RepartitionExec: partitioning=Hash([category@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[category@0 as category], aggr=[max(string_topk_view.val)], lim=[2]
@@ -268,11 +268,13 @@ NULL 0 0
c 1 2
# Regression tests for string max with ORDER BY ... LIMIT to ensure schema stability
+# Note: the NULL group has an all-NULL trace_id, so its max is NULL and ranks
+# first under DESC NULLS FIRST (previously the group was dropped: issue #23440)
query TT
select trace_id, max(trace_id) as max_trace from traces group by trace_id order by max_trace desc limit 2;
----
+NULL NULL
c c
-b b
query TT
explain select trace_id, max(trace_id) as max_trace from traces group by trace_id order by max_trace desc limit 2;
@@ -286,9 +288,9 @@ physical_plan
01)SortPreservingMergeExec: [max_trace@1 DESC], fetch=2
02)--ProjectionExec: expr=[trace_id@0 as trace_id, max(traces.trace_id)@1 as max_trace]
03)----SortExec: TopK(fetch=2), expr=[max(traces.trace_id)@1 DESC], preserve_partitioning=[true]
-04)------AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)], lim=[2]
+04)------AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)]
05)--------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=1
-06)----------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)], lim=[2]
+06)----------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)]
07)------------DataSourceExec: partitions=1, partition_sizes=[1]
@@ -303,15 +305,15 @@ AS SELECT
FROM traces;
query TT
-explain select trace_id, MAX(timestamp) from traces_utf8view group by trace_id order by MAX(timestamp) desc limit 4;
+explain select trace_id, MAX(timestamp) from traces_utf8view group by trace_id order by MAX(timestamp) desc nulls last limit 4;
----
logical_plan
-01)Sort: max(traces_utf8view.timestamp) DESC NULLS FIRST, fetch=4
+01)Sort: max(traces_utf8view.timestamp) DESC NULLS LAST, fetch=4
02)--Aggregate: groupBy=[[traces_utf8view.trace_id]], aggr=[[max(traces_utf8view.timestamp)]]
03)----TableScan: traces_utf8view projection=[trace_id, timestamp]
physical_plan
-01)SortPreservingMergeExec: [max(traces_utf8view.timestamp)@1 DESC], fetch=4
-02)--SortExec: TopK(fetch=4), expr=[max(traces_utf8view.timestamp)@1 DESC], preserve_partitioning=[true]
+01)SortPreservingMergeExec: [max(traces_utf8view.timestamp)@1 DESC NULLS LAST], fetch=4
+02)--SortExec: TopK(fetch=4), expr=[max(traces_utf8view.timestamp)@1 DESC NULLS LAST], preserve_partitioning=[true]
03)----AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces_utf8view.timestamp)], lim=[4]
04)------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=1
05)--------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces_utf8view.timestamp)], lim=[4]
@@ -329,15 +331,15 @@ AS SELECT
FROM traces;
query TT
-explain select trace_id, MAX(timestamp) from traces_largeutf8 group by trace_id order by MAX(timestamp) desc limit 4;
+explain select trace_id, MAX(timestamp) from traces_largeutf8 group by trace_id order by MAX(timestamp) desc nulls last limit 4;
----
logical_plan
-01)Sort: max(traces_largeutf8.timestamp) DESC NULLS FIRST, fetch=4
+01)Sort: max(traces_largeutf8.timestamp) DESC NULLS LAST, fetch=4
02)--Aggregate: groupBy=[[traces_largeutf8.trace_id]], aggr=[[max(traces_largeutf8.timestamp)]]
03)----TableScan: traces_largeutf8 projection=[trace_id, timestamp]
physical_plan
-01)SortPreservingMergeExec: [max(traces_largeutf8.timestamp)@1 DESC], fetch=4
-02)--SortExec: TopK(fetch=4), expr=[max(traces_largeutf8.timestamp)@1 DESC], preserve_partitioning=[true]
+01)SortPreservingMergeExec: [max(traces_largeutf8.timestamp)@1 DESC NULLS LAST], fetch=4
+02)--SortExec: TopK(fetch=4), expr=[max(traces_largeutf8.timestamp)@1 DESC NULLS LAST], preserve_partitioning=[true]
03)----AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces_largeutf8.timestamp)], lim=[4]
04)------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=1
05)--------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces_largeutf8.timestamp)], lim=[4]
@@ -585,3 +587,205 @@ drop table ids;
statement ok
drop table traces;
+
+#######
+# Regression tests for all-NULL groups in TopK aggregation (issues #23440, #22190):
+# a group whose aggregate inputs are all NULL must be emitted with a NULL
+# aggregate value instead of disappearing from the result
+#######
+statement ok
+CREATE TABLE t0 AS SELECT * FROM (VALUES ('gamma', CAST(NULL AS DOUBLE))) v(s, y);
+
+# MIN/MAX with NULLS FIRST must use regular aggregation because a group's
+# aggregate can transition from NULL to non-NULL and worsen its rank.
+query TT
+explain select s, max(y) as max_y from t0 group by s order by max_y desc nulls first limit 3;
+----
+logical_plan
+01)Sort: max_y DESC NULLS FIRST, fetch=3
+02)--Projection: t0.s, max(t0.y) AS max_y
+03)----Aggregate: groupBy=[[t0.s]], aggr=[[max(t0.y)]]
+04)------TableScan: t0 projection=[s, y]
+physical_plan
+01)ProjectionExec: expr=[s@0 as s, max(t0.y)@1 as max_y]
+02)--SortExec: TopK(fetch=3), expr=[max(t0.y)@1 DESC], preserve_partitioning=[false]
+03)----AggregateExec: mode=SinglePartitioned, gby=[s@0 as s], aggr=[max(t0.y)]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# issue #23440: single all-NULL group, MAX DESC NULLS FIRST LIMIT 3
+query R
+SELECT max_y FROM (SELECT s, MAX(y) AS max_y FROM t0 GROUP BY s) ORDER BY max_y DESC NULLS FIRST LIMIT 3;
+----
+NULL
+
+# issue #22190: single all-NULL group, MIN ASC NULLS LAST LIMIT 20
+query TT
+EXPLAIN SELECT min_y FROM (SELECT s, MIN(y) AS min_y FROM t0 GROUP BY s) ORDER BY min_y ASC NULLS LAST LIMIT 20;
+----
+logical_plan
+01)Sort: min_y ASC NULLS LAST, fetch=20
+02)--Projection: min(t0.y) AS min_y
+03)----Aggregate: groupBy=[[t0.s]], aggr=[[min(t0.y)]]
+04)------TableScan: t0 projection=[s, y]
+physical_plan
+01)SortExec: TopK(fetch=20), expr=[min_y@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--ProjectionExec: expr=[min(t0.y)@1 as min_y]
+03)----AggregateExec: mode=SinglePartitioned, gby=[s@0 as s], aggr=[min(t0.y)], lim=[20]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query R
+SELECT min_y FROM (SELECT s, MIN(y) AS min_y FROM t0 GROUP BY s) ORDER BY min_y ASC NULLS LAST LIMIT 20;
+----
+NULL
+
+# one all-NULL group and one valued group, limit larger than the group count:
+# both rows must be present
+statement ok
+CREATE TABLE topk_two_groups(s varchar, y bigint) AS VALUES
+('a', CAST(NULL AS BIGINT)),
+('b', 10),
+('b', 20);
+
+query TI
+select s, max_y from (select s, max(y) as max_y from topk_two_groups group by s) order by max_y desc nulls first limit 10;
+----
+a NULL
+b 20
+
+# 5 all-NULL groups with LIMIT 2: exactly 2 rows survive
+statement ok
+CREATE TABLE topk_five_nulls(s varchar, y bigint) AS VALUES
+('g1', CAST(NULL AS BIGINT)),
+('g2', CAST(NULL AS BIGINT)),
+('g3', CAST(NULL AS BIGINT)),
+('g4', CAST(NULL AS BIGINT)),
+('g5', CAST(NULL AS BIGINT));
+
+query I
+select max_y from (select s, max(y) as max_y from topk_five_nulls group by s) order by max_y desc nulls first limit 2;
+----
+NULL
+NULL
+
+# 2 all-NULL groups and 3 valued groups with LIMIT 4
+statement ok
+CREATE TABLE topk_mixed(s varchar, y bigint) AS VALUES
+('n1', CAST(NULL AS BIGINT)),
+('n2', CAST(NULL AS BIGINT)),
+('v1', 10),
+('v2', 20),
+('v3', 30);
+
+# DESC NULLS FIRST: both NULL groups rank before all values
+query I
+select max_y from (select s, max(y) as max_y from topk_mixed group by s) order by max_y desc nulls first limit 4;
+----
+NULL
+NULL
+30
+20
+
+# DESC NULLS LAST: NULL groups rank after all values
+query I
+select max_y from (select s, max(y) as max_y from topk_mixed group by s) order by max_y desc nulls last limit 4;
+----
+30
+20
+10
+NULL
+
+# an all-NULL group that later produces a value losing to the current top-k
+# must not be emitted with a NULL value
+statement ok
+CREATE TABLE topk_null_then_value(s varchar, y bigint) AS VALUES
+('g1', CAST(NULL AS BIGINT)),
+('g2', 10),
+('g1', 5);
+
+query I
+select max_y from (select s, max(y) as max_y from topk_null_then_value group by s) order by max_y desc nulls first limit 1;
+----
+10
+
+# single-row batches force NULL and non-NULL rows of the same group into
+# different batches: a -> 7, b -> 5, c -> NULL
+statement ok
+set datafusion.execution.batch_size = 1;
+
+statement ok
+CREATE TABLE topk_batches(s varchar, y bigint) AS VALUES
+('a', CAST(NULL AS BIGINT)),
+('b', 5),
+('a', 3),
+('c', CAST(NULL AS BIGINT)),
+('b', CAST(NULL AS BIGINT)),
+('a', 7);
+
+query TI
+select s, max_y from (select s, max(y) as max_y from topk_batches group by s) order by max_y desc nulls first limit 3;
+----
+c NULL
+a 7
+b 5
+
+# NULLS FIRST is not monotonic for MIN/MAX aggregation: a group initially
+# registered as NULL can later become valued, so a bounded TopK cannot safely
+# discard other NULL candidates. This must fall back to regular aggregation.
+statement ok
+set datafusion.execution.target_partitions = 1;
+
+statement ok
+CREATE TABLE topk_null_backfill(s varchar, y bigint) AS VALUES
+('a', CAST(NULL AS BIGINT)),
+('b', CAST(NULL AS BIGINT)),
+('c', CAST(NULL AS BIGINT)),
+('a', 5);
+
+query I
+select max_y from (select s, max(y) as max_y from topk_null_backfill group by s) order by max_y desc nulls first limit 2;
+----
+NULL
+NULL
+
+# An evicted valued group must not be re-registered and emitted as all-NULL.
+statement ok
+CREATE TABLE topk_evicted_then_null(s varchar, y bigint) AS VALUES
+('a', 10),
+('b', 20),
+('a', CAST(NULL AS BIGINT)),
+('c', CAST(NULL AS BIGINT));
+
+query TI
+select s, max_y from (select s, max(y) as max_y from topk_evicted_then_null group by s) order by max_y desc nulls first limit 1;
+----
+c NULL
+
+statement ok
+set datafusion.execution.batch_size = 8192;
+
+statement ok
+set datafusion.execution.target_partitions = 4;
+
+statement ok
+drop table topk_batches;
+
+statement ok
+drop table topk_evicted_then_null;
+
+statement ok
+drop table topk_null_backfill;
+
+statement ok
+drop table topk_null_then_value;
+
+statement ok
+drop table topk_mixed;
+
+statement ok
+drop table topk_five_nulls;
+
+statement ok
+drop table topk_two_groups;
+
+statement ok
+drop table t0;
diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt
index de493d6e4a..c4f2bcf2fa 100644
--- a/datafusion/sqllogictest/test_files/group_by.slt
+++ b/datafusion/sqllogictest/test_files/group_by.slt
@@ -4608,9 +4608,9 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [max(timestamp_table.t1)@1 DESC], fetch=4
02)--SortExec: TopK(fetch=4), expr=[max(timestamp_table.t1)@1 DESC], preserve_partitioning=[true]
-03)----AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[max(timestamp_table.t1)], lim=[4]
+03)----AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[max(timestamp_table.t1)]
04)------RepartitionExec: partitioning=Hash([c2@0], 8), input_partitions=8
-05)--------AggregateExec: mode=Partial, gby=[c2@1 as c2], aggr=[max(timestamp_table.t1)], lim=[4]
+05)--------AggregateExec: mode=Partial, gby=[c2@1 as c2], aggr=[max(timestamp_table.t1)]
06)----------RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=4
07)------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/group_by/timestamp_table/3.csv]]}, projection=[t1, c2], file_type=csv, has_header=true