Apache DataFusion SQL Query Engine
fix: Account for buffered rows in `RowCursorStream` (#25372)
Emily Matheysd173279fd8af
diff --git a/datafusion/physical-plan/src/sorts/cursor.rs b/datafusion/physical-plan/src/sorts/cursor.rsindex 700c95007c..60f612f640 100644--- a/datafusion/physical-plan/src/sorts/cursor.rs+++ b/datafusion/physical-plan/src/sorts/cursor.rs@@ -218,12 +218,13 @@ impl RowValues {/// Create a new [`RowValues`] from `rows` and a `reservation`/// that tracks its memory. There must be at least one row///- /// Panics if the reservation is not for exactly `rows.size()`- /// bytes or if `rows` is empty.+ /// The reservation must cover exactly `rows.size()` bytes, or be empty when+ /// the caller accounts for `rows` elsewhere for its whole lifetime.+ ///+ /// Panics if the reservation is neither of those, or if `rows` is empty.pub fn new(rows: Arc<Rows>, reservation: MemoryReservation) -> Self {- assert_eq!(- rows.size(),- reservation.size(),+ assert!(+ reservation.size() == 0 || reservation.size() == rows.size(),"memory reservation mismatch");let len = rows.num_rows();diff --git a/datafusion/physical-plan/src/sorts/stream.rs b/datafusion/physical-plan/src/sorts/stream.rsindex e2a840fded..d7bc628225 100644--- a/datafusion/physical-plan/src/sorts/stream.rs+++ b/datafusion/physical-plan/src/sorts/stream.rs@@ -94,25 +94,63 @@ impl FusedStreams {}}-/// An `Arc<Rows>` that can be reused+/// An `Arc<Rows>` that can be reused.+///+/// Owns the reservation covering every retained buffer for as long as it is+/// retained, so the cache is visible to the pool rather than held off-book.+/// A retained buffer keeps its capacity, so the cost is the high-water mark of+/// each stream, not the size of the batch currently in flight.#[derive(Debug)]struct ReusableRows {inner: Vec<Option<Arc<Rows>>>,+ reservation: MemoryReservation,}impl ReusableRows {// return a Rows for writing,// does not clone if the existing rows can be reused- fn take_next(&mut self, stream_idx: usize) -> Result<Rows> {- Arc::try_unwrap(self.inner[stream_idx].take().unwrap()).map_err(|_| {- internal_datafusion_err!(- "Rows from RowCursorStream is still in use by consumer"- )- })+ fn take_next(&mut self, stream_idx: usize, converter: &RowConverter) -> Result<Rows> {+ match self.inner[stream_idx].take() {+ Some(rows) => Arc::try_unwrap(rows).map_err(|_| {+ internal_datafusion_err!(+ "Rows from RowCursorStream is still in use by consumer"+ )+ }),+ // Nothing retained yet, or already released, so start over.+ None => Ok(converter.empty_rows(0, 0)),+ }}- // save the Rows- fn save(&mut self, stream_idx: usize, rows: &Arc<Rows>) {++ /// Account for a freshly built buffer, and retain it for reuse.+ ///+ /// The reservation is mandatory rather than best-effort. The buffer is live in the+ /// cursor whether or not this slot keeps a handle to it, so declining to reserve+ /// would hide it from the pool instead of avoiding it, and it frees nothing at this+ /// point either, since the cursor holds the same `Arc`. Retention on top of the+ /// reservation is free for the same reason. A pool that cannot cover the buffer+ /// fails the query here, as it did before the buffer was cached at all.+ fn save(&mut self, stream_idx: usize, rows: &Arc<Rows>) -> Result<()> {self.inner[stream_idx] = Some(Arc::clone(rows));+ let retained = self.retained_size();+ debug_assert!(retained >= self.reservation.size());+ if let Err(e) = self.reservation.try_resize(retained) {+ self.inner[stream_idx] = None;+ self.reservation.try_resize(self.retained_size())?;+ return Err(e);+ }+ Ok(())+ }++ // drop whatever a finished stream was holding+ fn release(&mut self, stream_idx: usize) {+ debug_assert!(self.reservation.size() >= self.retained_size());+ if let Some(rows) = self.inner[stream_idx].take() {+ self.reservation.shrink(rows.size());+ }+ }++ fn retained_size(&self) -> usize {+ self.inner.iter().flatten().map(|rows| rows.size()).sum()}}@@ -152,18 +190,18 @@ impl RowCursorStream {.collect::<Result<Vec<_>>>()?;let streams: Vec<_> = streams.into_iter().map(|s| s.fuse()).collect();+ let stream_count = streams.len();let converter = RowConverter::new(sort_fields)?;- let mut rows = Vec::with_capacity(streams.len());- for _ in &streams {- // Initialize each stream with an empty Rows- rows.push(Some(Arc::new(converter.empty_rows(0, 0))));- }+ let rows_reservation = reservation.new_empty();Ok(Self {converter,reservation,column_expressions: expressions.iter().map(|x| Arc::clone(&x.expr)).collect(),streams: FusedStreams(streams),- rows: ReusableRows { inner: rows },+ rows: ReusableRows {+ inner: vec![None; stream_count],+ reservation: rows_reservation,+ },})}@@ -175,7 +213,7 @@ impl RowCursorStream {let cols = evaluate_expressions_to_arrays(&self.column_expressions, batch)?;// At this point, ownership should of this Rows should be unique- let mut rows = self.rows.take_next(stream_idx)?;+ let mut rows = self.rows.take_next(stream_idx, &self.converter)?;rows.clear();@@ -184,12 +222,13 @@ impl RowCursorStream {let rows = Arc::new(rows);- self.rows.save(stream_idx, &rows);+ self.rows.save(stream_idx, &rows)?;- // track the memory in the newly created Rows.- let rows_reservation = self.reservation.new_empty();- rows_reservation.try_grow(rows.size())?;- Ok(RowValues::new(rows, rows_reservation))+ // `self.rows` now holds the reservation for this buffer unconditionally, and+ // holds it for at least as long as the cursor does. `take_next` cannot reclaim+ // the slot while the consumer still owns the `Arc`, so the cursor is handed an+ // empty reservation rather than accounting for the same bytes a second time.+ Ok(RowValues::new(rows, self.reservation.new_empty()))}}@@ -205,7 +244,12 @@ impl PartitionedStream for RowCursorStream {cx: &mut Context<'_>,stream_idx: usize,) -> Poll<Option<Self::Output>> {- Poll::Ready(ready!(self.streams.poll_next(cx, stream_idx)).map(|r| {+ let polled = ready!(self.streams.poll_next(cx, stream_idx));+ if polled.is_none() {+ // The stream is finished, so its retained buffer will never be reused.+ self.rows.release(stream_idx);+ }+ Poll::Ready(polled.map(|r| {r.and_then(|batch| {let cursor = self.convert_batch(&batch, stream_idx)?;Ok((cursor, batch))@@ -391,11 +435,15 @@ impl FusedIterator for IncrementalSortIterator {}#[cfg(test)]mod tests {use super::*;- use arrow::array::{AsArray, Int32Array};+ use crate::memory::MemoryStream;+ use arrow::array::{AsArray, Int32Array, StringArray};use arrow::datatypes::{DataType, Field, Int32Type};use arrow_schema::SchemaRef;use datafusion_common::DataFusionError;use datafusion_execution::RecordBatchStream;+ use datafusion_execution::memory_pool::{+ GreedyMemoryPool, MemoryConsumer, MemoryPool,+ };use datafusion_physical_expr::expressions::col;use futures::Stream;use std::pin::Pin;@@ -587,4 +635,276 @@ mod tests {Ok(())}++ /// Drives a `RowCursorStream` over several partitions and reports the pool+ /// reservation, so the tests below can watch it move.+ struct RowCursorHarness {+ stream: RowCursorStream,+ pool: Arc<dyn MemoryPool>,+ schema: SchemaRef,+ /// `(rows, str_width)` of each batch, per partition.+ shapes: Vec<Vec<(usize, usize)>>,+ }++ /// The converted `Rows` size depends only on `rows` and `str_width`.+ fn make_batch(+ schema: &SchemaRef,+ seq: usize,+ rows: usize,+ str_width: usize,+ ) -> RecordBatch {+ let base = (seq * rows) as i32;+ let a = Int32Array::from_iter_values((0..rows as i32).map(|i| base + i));+ let b = StringArray::from_iter_values((0..rows).map(|_| "x".repeat(str_width)));+ RecordBatch::try_new(Arc::clone(schema), vec![Arc::new(a), Arc::new(b)]).unwrap()+ }++ impl RowCursorHarness {+ /// `batches_per_partition` batches of `rows` rows each, per partition.+ /// The sort key spans two columns so this takes the `Rows` path rather+ /// than the specialized single-column `FieldCursorStream`.+ fn new(+ partitions: usize,+ batches_per_partition: usize,+ rows: usize,+ str_width: usize,+ ) -> Result<Self> {+ Self::with_shapes(vec![+ vec![(rows, str_width); batches_per_partition];+ partitions+ ])+ }++ /// One partition per entry, one `(rows, str_width)` batch per element.+ fn with_shapes(shapes: Vec<Vec<(usize, usize)>>) -> Result<Self> {+ let schema: SchemaRef = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Utf8, false),+ ]));++ let streams: Vec<SendableRecordBatchStream> = shapes+ .iter()+ .map(|partition| {+ let batches: Vec<RecordBatch> = partition+ .iter()+ .enumerate()+ .map(|(seq, &(rows, str_width))| {+ make_batch(&schema, seq, rows, str_width)+ })+ .collect();+ Box::pin(+ MemoryStream::try_new(batches, Arc::clone(&schema), None)+ .unwrap(),+ ) as SendableRecordBatchStream+ })+ .collect();++ let expressions = LexOrdering::new(vec![+ PhysicalSortExpr::new_default(col("a", &schema)?),+ PhysicalSortExpr::new_default(col("b", &schema)?),+ ])+ .unwrap();++ // Generous limit: this measures what is reserved, not what is refused.+ let pool: Arc<dyn MemoryPool> =+ Arc::new(GreedyMemoryPool::new(256 * 1024 * 1024));+ let reservation = MemoryConsumer::new("test").register(&pool);+ let stream =+ RowCursorStream::try_new(&schema, &expressions, streams, reservation)?;++ Ok(Self {+ stream,+ pool,+ schema,+ shapes,+ })+ }++ fn reserved(&self) -> usize {+ self.pool.reserved()+ }++ fn converter(&self) -> Result<RowConverter> {+ Ok(RowConverter::new(vec![+ SortField::new(DataType::Int32),+ SortField::new(DataType::Utf8),+ ])?)+ }++ /// `(converter size, retained Rows size)` when all batches share a shape.+ fn expected_sizes(&self) -> Result<(usize, usize)> {+ let converter = self.converter()?;+ Ok((converter.size(), self.expected_retained_sizes(0)?[0]))+ }++ /// `Rows::size()` after each of `stream_idx`'s batches, reusing one+ /// buffer like the stream does.+ fn expected_retained_sizes(&self, stream_idx: usize) -> Result<Vec<usize>> {+ let converter = self.converter()?;+ let mut rows = converter.empty_rows(0, 0);+ self.shapes[stream_idx]+ .iter()+ .enumerate()+ .map(|(seq, &(num_rows, str_width))| {+ let batch = make_batch(&self.schema, seq, num_rows, str_width);+ rows.clear();+ converter.append(&mut rows, batch.columns())?;+ Ok(rows.size())+ })+ .collect()+ }++ /// Polls `stream_idx`, handing back the cursor so the caller controls when+ /// it is dropped. `None` once the partition is finished.+ fn poll_cursor(&mut self, stream_idx: usize) -> Result<Option<RowValues>> {+ let waker = futures::task::noop_waker();+ let mut cx = Context::from_waker(&waker);+ match self.stream.poll_next(&mut cx, stream_idx) {+ Poll::Ready(Some(Ok((values, _batch)))) => Ok(Some(values)),+ Poll::Ready(Some(Err(e))) => Err(e),+ Poll::Ready(None) => Ok(None),+ Poll::Pending => unreachable!("MemoryStream is never pending"),+ }+ }++ /// Polls `stream_idx` and immediately drops the returned cursor, which is+ /// what the merge does once a cursor is exhausted. Returns `false` once+ /// the partition is finished.+ fn poll_and_drop_cursor(&mut self, stream_idx: usize) -> Result<bool> {+ let waker = futures::task::noop_waker();+ let mut cx = Context::from_waker(&waker);+ match self.stream.poll_next(&mut cx, stream_idx) {+ Poll::Ready(Some(Ok((values, _batch)))) => {+ drop(values);+ Ok(true)+ }+ Poll::Ready(Some(Err(e))) => Err(e),+ Poll::Ready(None) => Ok(false),+ Poll::Pending => unreachable!("MemoryStream is never pending"),+ }+ }+ }++ // `RowCursorStream` keeps one `Rows` buffer per partition alive between+ // polls so the allocation can be reused. That buffer is real memory, and it+ // outlives the cursor handed to the consumer: the merge drops a cursor as+ // soon as it is exhausted, but the buffer behind it stays cached.+ //+ // So dropping the cursor must not change what the pool reports. If it does,+ // the bytes still held by the cache have gone off the books and the pool is+ // under-counting a live allocation.+ #[test]+ fn dropping_a_cursor_does_not_unaccount_its_retained_buffer() -> Result<()> {+ let mut harness = RowCursorHarness::new(1, 4, 512, 64)?;+ let (converter_size, rows_size) = harness.expected_sizes()?;+ assert_eq!(harness.reserved(), 0);++ let cursor = harness.poll_cursor(0)?.expect("first batch");+ assert_eq!(harness.reserved(), converter_size + rows_size);++ // The cached `Rows` buffer outlives the cursor, so dropping the cursor+ // must not release its bytes.+ drop(cursor);+ assert_eq!(harness.reserved(), converter_size + rows_size);++ Ok(())+ }++ // The retained bytes are per partition, so what the pool reports has to+ // scale with partition count rather than staying flat at the size of+ // whichever batch happens to be in flight.+ #[test]+ fn retained_row_buffer_reservation_scales_with_partitions() -> Result<()> {+ for partitions in [2, 16] {+ let mut harness = RowCursorHarness::new(partitions, 4, 512, 64)?;+ let (converter_size, rows_size) = harness.expected_sizes()?;+ for stream_idx in 0..partitions {+ assert!(harness.poll_and_drop_cursor(stream_idx)?);+ }+ // One shared converter, plus one cached buffer per partition.+ assert_eq!(+ harness.reserved(),+ converter_size + partitions * rows_size,+ "{partitions} partitions"+ );+ }++ Ok(())+ }++ // The flip side: a partition that will never be polled again must hand its+ // buffer back. Otherwise the reservation only ever grows and a long merge+ // holds every partition's high-water mark until the whole stream drops.+ #[test]+ fn exhausted_partitions_release_their_retained_buffers() -> Result<()> {+ let partitions = 8;+ let batches_per_partition = 3;+ let mut harness =+ RowCursorHarness::new(partitions, batches_per_partition, 512, 64)?;+ let (converter_size, rows_size) = harness.expected_sizes()?;+ assert_eq!(harness.reserved(), 0);++ for stream_idx in 0..partitions {+ for _ in 0..batches_per_partition {+ assert!(harness.poll_and_drop_cursor(stream_idx)?);+ }+ }+ // Reusing a buffer for later batches of the same shape does not grow it.+ assert_eq!(harness.reserved(), converter_size + partitions * rows_size);++ // Poll each partition once more so it reports exhaustion, releasing+ // one buffer at a time.+ for stream_idx in 0..partitions {+ assert!(!harness.poll_and_drop_cursor(stream_idx)?);+ let still_retained = partitions - stream_idx - 1;+ assert_eq!(+ harness.reserved(),+ converter_size + still_retained * rows_size+ );+ }++ // Only the converter is left.+ assert_eq!(harness.reserved(), converter_size);++ Ok(())+ }++ // A reused buffer keeps its capacity, so the reservation should track each+ // partition's high-water mark and release it on exhaustion only.+ #[test]+ fn retained_buffer_reservation_tracks_high_water_mark() -> Result<()> {+ let small = (64, 8);+ let large = (2048, 128);+ let mut harness =+ RowCursorHarness::with_shapes(vec![vec![small, large, small], vec![small]])?;+ let converter_size = harness.converter()?.size();+ let growing = harness.expected_retained_sizes(0)?;+ let cached = harness.expected_retained_sizes(1)?[0];++ // The large batch grows the buffer; the next small one doesn't shrink it.+ assert!(growing[1] > growing[0]);+ assert_eq!(growing[2], growing[1]);++ // Partition 1 keeps its buffer cached throughout.+ assert!(harness.poll_and_drop_cursor(1)?);+ assert_eq!(harness.reserved(), converter_size + cached);++ for (seq, retained) in growing.iter().enumerate() {+ assert!(harness.poll_and_drop_cursor(0)?);+ assert_eq!(+ harness.reserved(),+ converter_size + cached + retained,+ "after batch {seq}"+ );+ }++ // Releasing partition 0 leaves partition 1 accounted for.+ assert!(!harness.poll_and_drop_cursor(0)?);+ assert_eq!(harness.reserved(), converter_size + cached);++ assert!(!harness.poll_and_drop_cursor(1)?);+ assert_eq!(harness.reserved(), converter_size);++ Ok(())+ }}