Skip to content

Apache DataFusion SQL Query Engine

fix: count the largest capacity seen for a shared buffer address (#27097)

Andy Grove407ab7ad2969
diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs
index ac3bc1fbcc..e092f700fa 100644
--- a/datafusion/common/src/utils/memory.rs
+++ b/datafusion/common/src/utils/memory.rs
@@ -5,6 +5,7 @@
//! This module provides a function to estimate the memory size of a HashTable prior to allocation
use crate::error::_exec_datafusion_err;
+use crate::hash_map::Entry;
use crate::{HashMap, HashSet, Result};
use arrow::array::types::{
ByteArrayType, ByteViewType, Int8Type, Int16Type, Int32Type, Int64Type,
@@ -226,9 +227,9 @@ impl RecordBatchMemoryCounter {
impl BufferVisitor for RecordBatchMemoryCounter {
fn visit(&mut self, buffer: &Buffer) {
- if self.counted_buffers.insert(buffer.data_ptr().addr()) {
- self.memory_usage += buffer.capacity();
- }
+ self.memory_usage += self
+ .counted_buffers
+ .insert(buffer.data_ptr().addr(), buffer.capacity());
}
}
@@ -237,8 +238,8 @@ impl BufferVisitor for RecordBatchMemoryCounter {
/// Operators that retain batches incrementally and drop them later (sort,
/// window, sort-merge join, TopK) count a batch when they retain it and
/// [`Self::uncount_batch`] it when they drop it. The tracker keeps a reference
-/// count per buffer: a buffer's capacity is added when its count goes from 0
-/// to 1 and subtracted when it returns to 0.
+/// count per buffer: it is charged the largest capacity seen while its count
+/// is positive, and released when its count returns to 0.
///
/// **Contract**: a batch must be uncounted *before* its buffers are dropped.
/// The tracker identifies buffers by their data-pointer address and does not
@@ -373,9 +374,9 @@ impl RecordBatchMemoryTracker {
/// Counting visitor: retains each buffer once more.
impl BufferVisitor for RecordBatchMemoryTracker {
fn visit(&mut self, buffer: &Buffer) {
- if self.counted_buffers.increment(buffer.data_ptr().addr()) {
- self.memory_usage += buffer.capacity();
- }
+ self.memory_usage += self
+ .counted_buffers
+ .increment(buffer.data_ptr().addr(), buffer.capacity());
}
}
@@ -384,9 +385,7 @@ struct Uncount<'a>(&'a mut RecordBatchMemoryTracker);
impl BufferVisitor for Uncount<'_> {
fn visit(&mut self, buffer: &Buffer) {
- if self.0.counted_buffers.decrement(buffer.data_ptr().addr()) {
- self.0.memory_usage -= buffer.capacity();
- }
+ self.0.memory_usage -= self.0.counted_buffers.decrement(buffer.data_ptr().addr());
}
}
@@ -604,18 +603,25 @@ fn array_children(array: &ArrayRef) -> Vec<&ArrayRef> {
}
/// Tracks a small number of buffers inline, avoiding a heap allocation for
-/// typical batches, and promotes to a hash set when more buffers are seen.
+/// typical batches, and promotes to a hash map when more buffers are seen.
+///
+/// Keeps the largest capacity seen per address: buffers built with
+/// `Buffer::from_custom_allocation`, such as FFI imports, can share an address
+/// while reporting different capacities.
#[derive(Debug)]
struct BufferIdSet {
inline: [Option<NonZero<usize>>; INLINE_BUFFER_IDS],
+ /// Counted capacity of each inline buffer, parallel to `inline`.
+ capacities: [usize; INLINE_BUFFER_IDS],
len: usize,
- overflow: Option<HashSet<NonZero<usize>>>,
+ overflow: Option<HashMap<NonZero<usize>, usize>>,
}
impl Default for BufferIdSet {
fn default() -> Self {
Self {
inline: [None; INLINE_BUFFER_IDS],
+ capacities: [0; INLINE_BUFFER_IDS],
len: 0,
overflow: None,
}
@@ -623,52 +629,67 @@ impl Default for BufferIdSet {
}
impl BufferIdSet {
- fn insert(&mut self, buffer_id: NonZero<usize>) -> bool {
+ /// Returns the bytes of `capacity` not already counted for `buffer_id`.
+ fn insert(&mut self, buffer_id: NonZero<usize>, capacity: usize) -> usize {
if let Some(overflow) = &mut self.overflow {
- return overflow.insert(buffer_id);
+ let counted = overflow.entry(buffer_id).or_insert(0);
+ return grow_counted_capacity(counted, capacity);
}
- if self.inline[..self.len].contains(&Some(buffer_id)) {
- return false;
+ for i in 0..self.len {
+ if self.inline[i] == Some(buffer_id) {
+ return grow_counted_capacity(&mut self.capacities[i], capacity);
+ }
}
if self.len < INLINE_BUFFER_IDS {
self.inline[self.len] = Some(buffer_id);
+ self.capacities[self.len] = capacity;
self.len += 1;
- return true;
+ return capacity;
}
- let mut overflow = HashSet::with_capacity(INLINE_BUFFER_IDS + 1);
- overflow.extend(self.inline.iter().flatten().copied());
- let inserted = overflow.insert(buffer_id);
+ let mut overflow = HashMap::with_capacity(INLINE_BUFFER_IDS + 1);
+ overflow.extend(self.inline.iter().flatten().copied().zip(self.capacities));
+ overflow.insert(buffer_id, capacity);
self.overflow = Some(overflow);
- inserted
+ capacity
}
}
+/// Raises `counted` to `capacity` if that is larger, returning the growth.
+#[inline(always)]
+fn grow_counted_capacity(counted: &mut usize, capacity: usize) -> usize {
+ let growth = capacity.saturating_sub(*counted);
+ *counted += growth;
+ growth
+}
+
/// Reference-counted buffer tracker with an inline fast path.
///
-/// Tracks buffer addresses with their reference counts. The first
-/// [`INLINE_BUFFER_IDS`] distinct buffers are stored in a fixed-size inline
-/// array to avoid heap allocation for typical batches. When more buffers are
-/// encountered, the tracker promotes to a heap-allocated `HashMap`.
+/// Tracks buffer addresses with their reference counts and counted capacities
+/// (the largest seen, as in [`BufferIdSet`]). The first [`INLINE_BUFFER_IDS`]
+/// distinct buffers are stored in fixed-size inline arrays to avoid heap
+/// allocation for typical batches. When more buffers are encountered, the
+/// tracker promotes to a heap-allocated `HashMap`.
///
-/// The inline path stores addresses and counts in **separate** arrays so that
-/// the hot `position()` scan over addresses touches a contiguous
+/// The inline path stores addresses, counts and capacities in **separate**
+/// arrays so that the hot scan over addresses touches a contiguous
/// `[Option<NonZero<usize>>; 16]` (128 bytes, two cache lines) without
-/// interleaved count words, matching the layout of the previous insert-only
-/// `BufferIdSet`.
+/// interleaved count words.
#[derive(Debug)]
struct BufferIdMap {
/// Inline storage: buffer addresses (None = empty slot).
ids: [Option<NonZero<usize>>; INLINE_BUFFER_IDS],
/// Inline storage: per-slot reference counts, parallel to `ids`.
counts: [u32; INLINE_BUFFER_IDS],
+ /// Inline storage: per-slot counted capacities, parallel to `ids`.
+ capacities: [usize; INLINE_BUFFER_IDS],
/// Number of occupied inline slots.
len: usize,
/// Overflow storage used when more than `INLINE_BUFFER_IDS` distinct
- /// buffers are tracked.
- overflow: Option<HashMap<NonZero<usize>, u32>>,
+ /// buffers are tracked: (reference count, counted capacity).
+ overflow: Option<HashMap<NonZero<usize>, (u32, usize)>>,
}
impl Default for BufferIdMap {
@@ -676,6 +697,7 @@ impl Default for BufferIdMap {
Self {
ids: [None; INLINE_BUFFER_IDS],
counts: [0; INLINE_BUFFER_IDS],
+ capacities: [0; INLINE_BUFFER_IDS],
len: 0,
overflow: None,
}
@@ -683,58 +705,55 @@ impl Default for BufferIdMap {
}
impl BufferIdMap {
- /// Increment the reference count for `buffer_id`. Returns `true` when the
- /// count went from 0 to 1 (i.e., the buffer is newly tracked).
+ /// Increment the reference count for `buffer_id`, returning the bytes of
+ /// `capacity` not already counted for it.
#[inline(always)]
- fn increment(&mut self, buffer_id: NonZero<usize>) -> bool {
+ fn increment(&mut self, buffer_id: NonZero<usize>, capacity: usize) -> usize {
if let Some(overflow) = &mut self.overflow {
- let count = overflow.entry(buffer_id).or_insert(0);
+ let (count, counted) = overflow.entry(buffer_id).or_insert((0, 0));
*count += 1;
- return *count == 1;
+ return grow_counted_capacity(counted, capacity);
}
// Search inline slots.
- if let Some(i) = self.ids[..self.len]
- .iter()
- .position(|id| *id == Some(buffer_id))
- {
- self.counts[i] += 1;
- return false; // Already tracked, count > 1 now.
+ for i in 0..self.len {
+ if self.ids[i] == Some(buffer_id) {
+ self.counts[i] += 1;
+ return grow_counted_capacity(&mut self.capacities[i], capacity);
+ }
}
// New buffer: try inline first.
if self.len < INLINE_BUFFER_IDS {
self.ids[self.len] = Some(buffer_id);
self.counts[self.len] = 1;
+ self.capacities[self.len] = capacity;
self.len += 1;
- return true;
+ return capacity;
}
// Promote to overflow.
let mut overflow = HashMap::with_capacity(INLINE_BUFFER_IDS + 1);
for i in 0..self.len {
if let Some(id) = self.ids[i] {
- overflow.insert(id, self.counts[i]);
+ overflow.insert(id, (self.counts[i], self.capacities[i]));
}
}
- overflow.insert(buffer_id, 1);
+ overflow.insert(buffer_id, (1, capacity));
self.overflow = Some(overflow);
- true
+ capacity
}
- /// Decrement the reference count for `buffer_id`. Returns `true` when the
- /// count reached zero (i.e., the buffer is no longer tracked). Returns
- /// `false` if the buffer was not tracked or its count is still positive.
- fn decrement(&mut self, buffer_id: NonZero<usize>) -> bool {
+ /// Decrement the reference count for `buffer_id`, returning its counted
+ /// capacity when the count reaches zero and 0 otherwise.
+ fn decrement(&mut self, buffer_id: NonZero<usize>) -> usize {
if let Some(overflow) = &mut self.overflow {
- if let Some(count) = overflow.get_mut(&buffer_id) {
- *count -= 1;
- if *count == 0 {
- overflow.remove(&buffer_id);
- return true;
- }
- }
- return false;
+ let Entry::Occupied(mut entry) = overflow.entry(buffer_id) else {
+ return 0;
+ };
+ let (count, _) = entry.get_mut();
+ *count -= 1;
+ return if *count == 0 { entry.remove().1 } else { 0 };
}
// Search inline slots.
@@ -743,20 +762,22 @@ impl BufferIdMap {
if self.ids[i] == target {
self.counts[i] -= 1;
if self.counts[i] == 0 {
+ let counted = self.capacities[i];
// Swap-remove: move last entry here.
self.len -= 1;
if i < self.len {
self.ids[i] = self.ids[self.len];
self.counts[i] = self.counts[self.len];
+ self.capacities[i] = self.capacities[self.len];
}
self.ids[self.len] = None;
- return true;
+ return counted;
}
- return false;
+ return 0;
}
}
- false
+ 0
}
}
@@ -1229,6 +1250,93 @@ mod record_batch_tests {
assert_eq!(counter.count_batch(&batch), 0);
}
+ /// Arrays over one allocation with different capacities, as
+ /// `arrow::ffi::from_ffi` imports slices of one batch.
+ fn custom_allocation_arrays(lens: &[usize]) -> Vec<ArrayRef> {
+ use arrow::buffer::ScalarBuffer;
+ use std::ptr::NonNull;
+
+ let max_len = lens.iter().copied().max().unwrap_or(0);
+ let owner: Arc<Vec<i32>> = Arc::new((0..max_len as i32).collect());
+ let ptr = NonNull::new(owner.as_ptr().cast::<u8>().cast_mut()).unwrap();
+ lens.iter()
+ .map(|&len| {
+ // SAFETY: `owner` holds `len` values at `ptr`; the buffer keeps it alive.
+ let buffer = unsafe {
+ Buffer::from_custom_allocation(
+ ptr,
+ len * size_of::<i32>(),
+ Arc::clone(&owner) as _,
+ )
+ };
+ Arc::new(Int32Array::new(ScalarBuffer::new(buffer, 0, len), None))
+ as ArrayRef
+ })
+ .collect()
+ }
+
+ /// Arrays that fill the inline slots, so the next buffer promotes to overflow.
+ fn inline_filler_arrays() -> Vec<ArrayRef> {
+ (0..INLINE_BUFFER_IDS)
+ .map(|i| Arc::new(Int32Array::from(vec![i as i32])) as ArrayRef)
+ .collect()
+ }
+
+ #[test]
+ fn test_record_batch_memory_counter_counts_largest_capacity_at_address() {
+ let (small, large) = (16 * size_of::<i32>(), 1024 * size_of::<i32>());
+ for promote in [false, true] {
+ for (lens, expected) in [
+ ([16, 1024], (small, large - small)),
+ ([1024, 16], (large, 0)),
+ ] {
+ let arrays = custom_allocation_arrays(&lens);
+ let mut counter = RecordBatchMemoryCounter::new();
+ if promote {
+ for array in inline_filler_arrays() {
+ counter.count_array(array.as_ref());
+ }
+ }
+ let base = counter.memory_usage();
+
+ let first = counter.count_array(arrays[0].as_ref());
+ let second = counter.count_array(arrays[1].as_ref());
+ assert_eq!((first, second), expected);
+ assert_eq!(counter.memory_usage() - base, large);
+ assert_eq!(counter.counted_buffers.overflow.is_some(), promote);
+ }
+ }
+ }
+
+ #[test]
+ fn test_uncount_releases_largest_capacity_at_address() {
+ let (small, large) = (16 * size_of::<i32>(), 1024 * size_of::<i32>());
+ for promote in [false, true] {
+ let arrays = custom_allocation_arrays(&[16, 1024]);
+ let mut tracker = RecordBatchMemoryTracker::new();
+ let fillers = inline_filler_arrays();
+ if promote {
+ for array in &fillers {
+ tracker.count_array(array.as_ref());
+ }
+ }
+ let base = tracker.memory_usage();
+
+ assert_eq!(tracker.count_array(arrays[0].as_ref()), small);
+ assert_eq!(tracker.count_array(arrays[1].as_ref()), large - small);
+ assert_eq!(tracker.counted_buffers.overflow.is_some(), promote);
+ // Released at its largest capacity once the last array goes.
+ assert_eq!(tracker.uncount_array(arrays[0].as_ref()), 0);
+ assert_eq!(tracker.uncount_array(arrays[1].as_ref()), large);
+ assert_eq!(tracker.memory_usage(), base);
+
+ for array in &fillers {
+ tracker.uncount_array(array.as_ref());
+ }
+ assert_eq!(tracker.memory_usage(), 0);
+ }
+ }
+
#[test]
fn test_array_memory_size_matches_array_data_layouts() {
let list_field = Arc::new(Field::new_list_field(DataType::Int32, true));
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index 5c91c3a834..17e66ef339 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -54,8 +54,8 @@ use datafusion_common::utils::{
get_available_parallelism, memory::RecordBatchMemoryCounter,
};
use datafusion_common::{
- DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,
- unwrap_or_internal_err,
+ DataFusionError, Result, assert_eq_or_internal_err, assert_or_internal_err,
+ internal_datafusion_err, unwrap_or_internal_err,
};
use datafusion_common_runtime::SpawnedTask;
use datafusion_execution::TaskContext;
@@ -1017,9 +1017,11 @@ impl ExternalSorter {
{
let reserved_bytes = get_reserved_bytes_for_record_batch(&batch)?;
match &shared_input {
- None => assert_eq!(reserved_bytes, reservation.size()),
+ None => assert_eq_or_internal_err!(reserved_bytes, reservation.size()),
// `shared_input` covers every buffer of every run's input
- Some(shared) => assert!(reservation.size() + shared.size() >= reserved_bytes),
+ Some(shared) => assert_or_internal_err!(
+ reservation.size() + shared.size() >= reserved_bytes
+ ),
}
let expressions = self.expr.clone();
@@ -2535,6 +2537,60 @@ mod tests {
use insta::assert_snapshot;
use tokio::sync::Notify;
+ /// Runs sharing a buffer address with different capacities, as FFI-imported
+ /// slices do. <https://github.com/apache/datafusion/issues/27069>
+ #[tokio::test]
+ async fn sort_runs_sharing_address_with_different_capacities() -> Result<()> {
+ use arrow::buffer::{Buffer, ScalarBuffer};
+ use std::ptr::NonNull;
+
+ let owner: Arc<Vec<i32>> = Arc::new((0..1024).rev().collect());
+ let ptr = NonNull::new(owner.as_ptr().cast::<u8>().cast_mut()).unwrap();
+ let batch = |len: usize| -> Result<RecordBatch> {
+ // SAFETY: `owner` holds `len` values at `ptr`; the buffer keeps it alive.
+ let buffer = unsafe {
+ Buffer::from_custom_allocation(
+ ptr,
+ len * size_of::<i32>(),
+ Arc::clone(&owner) as _,
+ )
+ };
+ let array = Int32Array::new(ScalarBuffer::new(buffer, 0, len), None);
+ Ok(RecordBatch::try_from_iter([(
+ "a",
+ Arc::new(array) as ArrayRef,
+ )])?)
+ };
+ // Smaller capacity first, so it is the first one seen for the address.
+ let batches = vec![batch(16)?, batch(1024)?];
+ let schema = batches[0].schema();
+ let input = TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;
+ let sort = SortExec::new(
+ [PhysicalSortExpr::new_default(col("a", &schema)?)].into(),
+ input,
+ );
+ // Keep each batch as its own run.
+ let context = Arc::new(TaskContext::default().with_session_config(
+ SessionConfig::new().with_sort_in_place_threshold_bytes(0),
+ ));
+
+ let output = collect(Arc::new(sort), context).await?;
+ let values = output
+ .iter()
+ .flat_map(|batch| {
+ batch
+ .column(0)
+ .as_primitive::<Int32Type>()
+ .values()
+ .to_vec()
+ })
+ .collect::<Vec<_>>();
+ let mut expected = (0..1024).chain(1008..1024).collect::<Vec<i32>>();
+ expected.sort_unstable();
+ assert_eq!(values, expected);
+ Ok(())
+ }
+
#[tokio::test]
async fn sort_run_records_compute_before_consumption() -> Result<()> {
let batch = RecordBatch::try_from_iter([(