Skip to content

Apache DataFusion SQL Query Engine

Fix sort merge interleave overflow (#20922)

xudong.w07ddfd7a22bf
diff --git a/datafusion/physical-plan/src/sorts/builder.rs b/datafusion/physical-plan/src/sorts/builder.rs
index a462b83205..73386212a2 100644
--- a/datafusion/physical-plan/src/sorts/builder.rs
+++ b/datafusion/physical-plan/src/sorts/builder.rs
@@ -16,11 +16,16 @@
// under the License.
use crate::spill::get_record_batch_memory_size;
+use arrow::array::ArrayRef;
use arrow::compute::interleave;
use arrow::datatypes::SchemaRef;
+use arrow::error::ArrowError;
use arrow::record_batch::RecordBatch;
-use datafusion_common::Result;
+use datafusion_common::{DataFusionError, Result};
use datafusion_execution::memory_pool::MemoryReservation;
+use log::warn;
+use std::any::Any;
+use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::Arc;
#[derive(Debug, Copy, Clone, Default)]
@@ -126,49 +131,60 @@ impl BatchBuilder {
&self.schema
}
- /// Drains the in_progress row indexes, and builds a new RecordBatch from them
- ///
- /// Will then drop any batches for which all rows have been yielded to the output
- ///
- /// Returns `None` if no pending rows
- pub fn build_record_batch(&mut self) -> Result<Option<RecordBatch>> {
- if self.is_empty() {
- return Ok(None);
- }
-
- let columns = (0..self.schema.fields.len())
+ /// Try to interleave all columns using the given index slice.
+ fn try_interleave_columns(
+ &self,
+ indices: &[(usize, usize)],
+ ) -> Result<Vec<ArrayRef>> {
+ (0..self.schema.fields.len())
.map(|column_idx| {
let arrays: Vec<_> = self
.batches
.iter()
.map(|(_, batch)| batch.column(column_idx).as_ref())
.collect();
- Ok(interleave(&arrays, &self.indices)?)
+ recover_offset_overflow_from_panic(|| interleave(&arrays, indices))
})
- .collect::<Result<Vec<_>>>()?;
-
- self.indices.clear();
-
- // New cursors are only created once the previous cursor for the stream
- // is finished. This means all remaining rows from all but the last batch
- // for each stream have been yielded to the newly created record batch
- //
- // We can therefore drop all but the last batch for each stream
- let mut batch_idx = 0;
- let mut retained = 0;
- self.batches.retain(|(stream_idx, batch)| {
- let stream_cursor = &mut self.cursors[*stream_idx];
- let retain = stream_cursor.batch_idx == batch_idx;
- batch_idx += 1;
-
- if retain {
- stream_cursor.batch_idx = retained;
- retained += 1;
- } else {
- self.batches_mem_used -= get_record_batch_memory_size(batch);
- }
- retain
- });
+ .collect::<Result<Vec<_>>>()
+ }
+
+ /// Builds a record batch from the first `rows_to_emit` buffered rows.
+ fn finish_record_batch(
+ &mut self,
+ rows_to_emit: usize,
+ columns: Vec<ArrayRef>,
+ ) -> Result<RecordBatch> {
+ // Remove consumed indices, keeping any remaining for the next call.
+ self.indices.drain(..rows_to_emit);
+
+ // Only clean up fully-consumed batches when all indices are drained,
+ // because remaining indices may still reference earlier batches.
+ // In the overflow/partial-emit case this may retain some extra memory
+ // across a few drain polls, but avoids costly index scanning on the
+ // hot path. The retention is bounded and short-lived since leftover
+ // rows are drained over subsequent polls.
+ if self.indices.is_empty() {
+ // New cursors are only created once the previous cursor for the stream
+ // is finished. This means all remaining rows from all but the last batch
+ // for each stream have been yielded to the newly created record batch
+ //
+ // We can therefore drop all but the last batch for each stream
+ let mut batch_idx = 0;
+ let mut retained = 0;
+ self.batches.retain(|(stream_idx, batch)| {
+ let stream_cursor = &mut self.cursors[*stream_idx];
+ let retain = stream_cursor.batch_idx == batch_idx;
+ batch_idx += 1;
+
+ if retain {
+ stream_cursor.batch_idx = retained;
+ retained += 1;
+ } else {
+ self.batches_mem_used -= get_record_batch_memory_size(batch);
+ }
+ retain
+ });
+ }
// Release excess memory back to the pool, but never shrink below
// initial_reservation to maintain the anti-starvation guarantee
@@ -178,10 +194,27 @@ impl BatchBuilder {
self.reservation.shrink(self.reservation.size() - target);
}
- Ok(Some(RecordBatch::try_new(
- Arc::clone(&self.schema),
- columns,
- )?))
+ RecordBatch::try_new(Arc::clone(&self.schema), columns).map_err(Into::into)
+ }
+
+ /// Drains the in_progress row indexes, and builds a new RecordBatch from them
+ ///
+ /// Will then drop any batches for which all rows have been yielded to the output.
+ /// If an offset overflow occurs (e.g. string/list offsets exceed i32::MAX),
+ /// retries with progressively fewer rows until it succeeds.
+ ///
+ /// Returns `None` if no pending rows
+ pub fn build_record_batch(&mut self) -> Result<Option<RecordBatch>> {
+ if self.is_empty() {
+ return Ok(None);
+ }
+
+ let (rows_to_emit, columns) =
+ retry_interleave(self.indices.len(), self.indices.len(), |rows_to_emit| {
+ self.try_interleave_columns(&self.indices[..rows_to_emit])
+ })?;
+
+ Ok(Some(self.finish_record_batch(rows_to_emit, columns)?))
}
}
@@ -200,3 +233,146 @@ pub(crate) fn try_grow_reservation_to_at_least(
}
Ok(())
}
+
+/// Returns true if the error is an Arrow offset overflow.
+fn is_offset_overflow(e: &DataFusionError) -> bool {
+ matches!(
+ e,
+ DataFusionError::ArrowError(boxed, _)
+ if matches!(boxed.as_ref(), ArrowError::OffsetOverflowError(_))
+ )
+}
+
+fn offset_overflow_error() -> DataFusionError {
+ DataFusionError::ArrowError(Box::new(ArrowError::OffsetOverflowError(0)), None)
+}
+
+fn recover_offset_overflow_from_panic<T, F>(f: F) -> Result<T>
+where
+ F: FnOnce() -> std::result::Result<T, ArrowError>,
+{
+ // Arrow's interleave can panic on i32 offset overflow with
+ // `.expect("overflow")` / `.expect("offset overflow")`.
+ // Catch only those specific panics so the caller can retry
+ // with fewer rows while unrelated defects still unwind.
+ //
+ // TODO: remove once arrow-rs#9549 lands — interleave will return
+ // OffsetOverflowError directly instead of panicking.
+ match catch_unwind(AssertUnwindSafe(f)) {
+ Ok(result) => Ok(result?),
+ Err(panic_payload) => {
+ if is_arrow_offset_overflow_panic(panic_payload.as_ref()) {
+ Err(offset_overflow_error())
+ } else {
+ std::panic::resume_unwind(panic_payload);
+ }
+ }
+ }
+}
+
+fn retry_interleave<T, F>(
+ mut rows_to_emit: usize,
+ total_rows: usize,
+ mut interleave: F,
+) -> Result<(usize, T)>
+where
+ F: FnMut(usize) -> Result<T>,
+{
+ loop {
+ match interleave(rows_to_emit) {
+ Ok(value) => return Ok((rows_to_emit, value)),
+ Err(e) if is_offset_overflow(&e) => {
+ rows_to_emit /= 2;
+ if rows_to_emit == 0 {
+ return Err(e);
+ }
+ warn!(
+ "Interleave offset overflow with {total_rows} rows, retrying with {rows_to_emit}"
+ );
+ }
+ Err(e) => return Err(e),
+ }
+ }
+}
+
+fn panic_message(payload: &(dyn Any + Send)) -> Option<&str> {
+ if let Some(msg) = payload.downcast_ref::<&str>() {
+ return Some(msg);
+ }
+ if let Some(msg) = payload.downcast_ref::<String>() {
+ return Some(msg.as_str());
+ }
+ None
+}
+
+/// Returns true if a caught panic payload matches the Arrow offset overflows
+/// raised by interleave's offset builders.
+fn is_arrow_offset_overflow_panic(payload: &(dyn Any + Send)) -> bool {
+ matches!(panic_message(payload), Some("overflow" | "offset overflow"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::error::ArrowError;
+
+ #[test]
+ fn test_retry_interleave_halves_rows_until_success() {
+ let mut attempts = Vec::new();
+
+ let (rows_to_emit, result) = retry_interleave(4, 4, |rows_to_emit| {
+ attempts.push(rows_to_emit);
+ if rows_to_emit > 1 {
+ Err(offset_overflow_error())
+ } else {
+ Ok("ok")
+ }
+ })
+ .unwrap();
+
+ assert_eq!(rows_to_emit, 1);
+ assert_eq!(result, "ok");
+ assert_eq!(attempts, vec![4, 2, 1]);
+ }
+
+ #[test]
+ fn test_recover_offset_overflow_from_panic() {
+ let error = recover_offset_overflow_from_panic(
+ || -> std::result::Result<(), ArrowError> { panic!("offset overflow") },
+ )
+ .unwrap_err();
+
+ assert!(is_offset_overflow(&error));
+ }
+
+ #[test]
+ fn test_recover_offset_overflow_from_panic_rethrows_unrelated_panics() {
+ let panic_payload = catch_unwind(AssertUnwindSafe(|| {
+ let _ = recover_offset_overflow_from_panic(
+ || -> std::result::Result<(), ArrowError> { panic!("capacity overflow") },
+ );
+ }));
+
+ assert!(panic_payload.is_err());
+ }
+
+ #[test]
+ fn test_is_arrow_offset_overflow_panic() {
+ let overflow = Box::new("overflow") as Box<dyn Any + Send>;
+ assert!(is_arrow_offset_overflow_panic(overflow.as_ref()));
+
+ let offset_overflow =
+ Box::new(String::from("offset overflow")) as Box<dyn Any + Send>;
+ assert!(is_arrow_offset_overflow_panic(offset_overflow.as_ref()));
+
+ let capacity_overflow = Box::new("capacity overflow") as Box<dyn Any + Send>;
+ assert!(!is_arrow_offset_overflow_panic(capacity_overflow.as_ref()));
+
+ let arithmetic_overflow =
+ Box::new(String::from("attempt to multiply with overflow"))
+ as Box<dyn Any + Send>;
+ assert!(!is_arrow_offset_overflow_panic(
+ arithmetic_overflow.as_ref()
+ ));
+ }
+}
diff --git a/datafusion/physical-plan/src/sorts/merge.rs b/datafusion/physical-plan/src/sorts/merge.rs
index 272816251d..c29933535a 100644
--- a/datafusion/physical-plan/src/sorts/merge.rs
+++ b/datafusion/physical-plan/src/sorts/merge.rs
@@ -53,6 +53,14 @@ pub(crate) struct SortPreservingMergeStream<C: CursorValues> {
/// `fetch` limit.
done: bool,
+ /// Whether buffered rows should be drained after `done` is set.
+ ///
+ /// This is enabled when we stop because the `fetch` limit has been
+ /// reached, allowing partial batches left over after overflow handling to
+ /// be emitted on subsequent polls. It remains disabled for terminal
+ /// errors so the stream does not yield data after returning `Err`.
+ drain_in_progress_on_done: bool,
+
/// A loser tree that always produces the minimum cursor
///
/// Node 0 stores the top winner, Nodes 1..num_streams store
@@ -164,6 +172,7 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
streams,
metrics,
done: false,
+ drain_in_progress_on_done: false,
cursors: (0..stream_count).map(|_| None).collect(),
prev_cursors: (0..stream_count).map(|_| None).collect(),
round_robin_tie_breaker_mode: false,
@@ -203,11 +212,28 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
}
}
+ fn emit_in_progress_batch(&mut self) -> Result<Option<RecordBatch>> {
+ let rows_before = self.in_progress.len();
+ let result = self.in_progress.build_record_batch();
+ self.produced += rows_before - self.in_progress.len();
+ result
+ }
+
fn poll_next_inner(
&mut self,
cx: &mut Context<'_>,
) -> Poll<Option<Result<RecordBatch>>> {
if self.done {
+ // When `build_record_batch()` hits an i32 offset overflow (e.g.
+ // combined string offsets exceed 2 GB), it emits a partial batch
+ // and keeps the remaining rows in `self.in_progress.indices`.
+ // Drain those leftover rows before terminating the stream,
+ // otherwise they would be silently dropped.
+ // Repeated overflows are fine — each poll emits another partial
+ // batch until `in_progress` is fully drained.
+ if self.drain_in_progress_on_done && !self.in_progress.is_empty() {
+ return Poll::Ready(self.emit_in_progress_batch().transpose());
+ }
return Poll::Ready(None);
}
// Once all partitions have set their corresponding cursors for the loser tree,
@@ -283,14 +309,13 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
// stop sorting if fetch has been reached
if self.fetch_reached() {
self.done = true;
+ self.drain_in_progress_on_done = true;
} else if self.in_progress.len() < self.batch_size {
continue;
}
}
- self.produced += self.in_progress.len();
-
- return Poll::Ready(self.in_progress.build_record_batch().transpose());
+ return Poll::Ready(self.emit_in_progress_batch().transpose());
}
}
@@ -542,3 +567,95 @@ impl<C: CursorValues + Unpin> RecordBatchStream for SortPreservingMergeStream<C>
Arc::clone(self.in_progress.schema())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::metrics::ExecutionPlanMetricsSet;
+ use crate::sorts::stream::PartitionedStream;
+ use arrow::array::Int32Array;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_execution::memory_pool::{
+ MemoryConsumer, MemoryPool, UnboundedMemoryPool,
+ };
+ use futures::task::noop_waker_ref;
+ use std::cmp::Ordering;
+
+ #[derive(Debug)]
+ struct EmptyPartitionedStream;
+
+ impl PartitionedStream for EmptyPartitionedStream {
+ type Output = Result<(DummyValues, RecordBatch)>;
+
+ fn partitions(&self) -> usize {
+ 1
+ }
+
+ fn poll_next(
+ &mut self,
+ _cx: &mut Context<'_>,
+ _stream_idx: usize,
+ ) -> Poll<Option<Self::Output>> {
+ Poll::Ready(None)
+ }
+ }
+
+ #[derive(Debug)]
+ struct DummyValues;
+
+ impl CursorValues for DummyValues {
+ fn len(&self) -> usize {
+ 0
+ }
+
+ fn eq(_l: &Self, _l_idx: usize, _r: &Self, _r_idx: usize) -> bool {
+ unreachable!("done-path test should not compare cursors")
+ }
+
+ fn eq_to_previous(_cursor: &Self, _idx: usize) -> bool {
+ unreachable!("done-path test should not compare cursors")
+ }
+
+ fn compare(_l: &Self, _l_idx: usize, _r: &Self, _r_idx: usize) -> Ordering {
+ unreachable!("done-path test should not compare cursors")
+ }
+ }
+
+ #[test]
+ fn test_done_drains_buffered_rows() {
+ let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));
+ let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
+ let reservation = MemoryConsumer::new("test").register(&pool);
+ let metrics = ExecutionPlanMetricsSet::new();
+
+ let mut stream = SortPreservingMergeStream::<DummyValues>::new(
+ Box::new(EmptyPartitionedStream),
+ Arc::clone(&schema),
+ BaselineMetrics::new(&metrics, 0),
+ 16,
+ Some(1),
+ reservation,
+ true,
+ );
+
+ let batch =
+ RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))])
+ .unwrap();
+ stream.in_progress.push_batch(0, batch).unwrap();
+ stream.in_progress.push_row(0);
+ stream.done = true;
+ stream.drain_in_progress_on_done = true;
+
+ let waker = noop_waker_ref();
+ let mut cx = Context::from_waker(waker);
+
+ match stream.poll_next_inner(&mut cx) {
+ Poll::Ready(Some(Ok(batch))) => assert_eq!(batch.num_rows(), 1),
+ other => {
+ panic!("expected buffered rows to be drained after done, got {other:?}")
+ }
+ }
+ assert!(stream.in_progress.is_empty());
+ assert!(matches!(stream.poll_next_inner(&mut cx), Poll::Ready(None)));
+ }
+}
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index b1ee5b4d5e..1e60c391f5 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -491,6 +491,7 @@ mod tests {
.with_session_config(config);
Ok(Arc::new(task_ctx))
}
+
// The number in the function is highly related to the memory limit we are testing,
// any change of the constant should be aware of
fn generate_spm_for_round_robin_tie_breaker(
@@ -1539,4 +1540,59 @@ mod tests {
Err(_) => exec_err!("SortPreservingMerge caused a deadlock"),
}
}
+
+ #[tokio::test]
+ async fn test_sort_merge_stops_after_error_with_buffered_rows() -> Result<()> {
+ let task_ctx = Arc::new(TaskContext::default());
+ let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));
+ let sort: LexOrdering = [PhysicalSortExpr::new_default(Arc::new(Column::new(
+ "i", 0,
+ ))
+ as Arc<dyn PhysicalExpr>)]
+ .into();
+
+ let mut stream0 = RecordBatchReceiverStream::builder(Arc::clone(&schema), 2);
+ let tx0 = stream0.tx();
+ let schema0 = Arc::clone(&schema);
+ stream0.spawn(async move {
+ let batch =
+ RecordBatch::try_new(schema0, vec![Arc::new(Int32Array::from(vec![1]))])?;
+ tx0.send(Ok(batch)).await.unwrap();
+ tx0.send(exec_err!("stream failure")).await.unwrap();
+ Ok(())
+ });
+
+ let mut stream1 = RecordBatchReceiverStream::builder(Arc::clone(&schema), 1);
+ let tx1 = stream1.tx();
+ let schema1 = Arc::clone(&schema);
+ stream1.spawn(async move {
+ let batch =
+ RecordBatch::try_new(schema1, vec![Arc::new(Int32Array::from(vec![2]))])?;
+ tx1.send(Ok(batch)).await.unwrap();
+ Ok(())
+ });
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let reservation =
+ MemoryConsumer::new("test").register(&task_ctx.runtime_env().memory_pool);
+
+ let mut merge_stream = StreamingMergeBuilder::new()
+ .with_streams(vec![stream0.build(), stream1.build()])
+ .with_schema(Arc::clone(&schema))
+ .with_expressions(&sort)
+ .with_metrics(BaselineMetrics::new(&metrics, 0))
+ .with_batch_size(task_ctx.session_config().batch_size())
+ .with_fetch(None)
+ .with_reservation(reservation)
+ .build()?;
+
+ let first = merge_stream.next().await.unwrap();
+ assert!(first.is_err(), "expected merge stream to surface the error");
+ assert!(
+ merge_stream.next().await.is_none(),
+ "merge stream yielded data after returning an error"
+ );
+
+ Ok(())
+ }
}