Skip to content

Apache DataFusion SQL Query Engine

Follow-up: remove interleave panic recovery after Arrow 58.1.0 (#21436)

xudong.w603bfb4ed873
diff --git a/datafusion/physical-plan/src/sorts/builder.rs b/datafusion/physical-plan/src/sorts/builder.rs
index 73386212a2..75eb2ff980 100644
--- a/datafusion/physical-plan/src/sorts/builder.rs
+++ b/datafusion/physical-plan/src/sorts/builder.rs
@@ -24,8 +24,6 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::{DataFusionError, Result};
use datafusion_execution::memory_pool::MemoryReservation;
use log::warn;
-use std::any::Any;
-use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::Arc;
#[derive(Debug, Copy, Clone, Default)]
@@ -143,7 +141,9 @@ impl BatchBuilder {
.iter()
.map(|(_, batch)| batch.column(column_idx).as_ref())
.collect();
- recover_offset_overflow_from_panic(|| interleave(&arrays, indices))
+ // Arrow 58.1.0+ returns OffsetOverflowError directly from
+ // interleave, allowing retry_interleave to shrink the batch.
+ interleave(&arrays, indices).map_err(Into::into)
})
.collect::<Result<Vec<_>>>()
}
@@ -243,33 +243,11 @@ fn is_offset_overflow(e: &DataFusionError) -> bool {
)
}
+#[cfg(test)]
fn offset_overflow_error() -> DataFusionError {
DataFusionError::ArrowError(Box::new(ArrowError::OffsetOverflowError(0)), None)
}
-fn recover_offset_overflow_from_panic<T, F>(f: F) -> Result<T>
-where
- F: FnOnce() -> std::result::Result<T, ArrowError>,
-{
- // Arrow's interleave can panic on i32 offset overflow with
- // `.expect("overflow")` / `.expect("offset overflow")`.
- // Catch only those specific panics so the caller can retry
- // with fewer rows while unrelated defects still unwind.
- //
- // TODO: remove once arrow-rs#9549 lands — interleave will return
- // OffsetOverflowError directly instead of panicking.
- match catch_unwind(AssertUnwindSafe(f)) {
- Ok(result) => Ok(result?),
- Err(panic_payload) => {
- if is_arrow_offset_overflow_panic(panic_payload.as_ref()) {
- Err(offset_overflow_error())
- } else {
- std::panic::resume_unwind(panic_payload);
- }
- }
- }
-}
-
fn retry_interleave<T, F>(
mut rows_to_emit: usize,
total_rows: usize,
@@ -281,6 +259,7 @@ where
loop {
match interleave(rows_to_emit) {
Ok(value) => return Ok((rows_to_emit, value)),
+ // Only offset overflow is recoverable by emitting fewer rows.
Err(e) if is_offset_overflow(&e) => {
rows_to_emit /= 2;
if rows_to_emit == 0 {
@@ -295,26 +274,34 @@ where
}
}
-fn panic_message(payload: &(dyn Any + Send)) -> Option<&str> {
- if let Some(msg) = payload.downcast_ref::<&str>() {
- return Some(msg);
- }
- if let Some(msg) = payload.downcast_ref::<String>() {
- return Some(msg.as_str());
- }
- None
-}
-
-/// Returns true if a caught panic payload matches the Arrow offset overflows
-/// raised by interleave's offset builders.
-fn is_arrow_offset_overflow_panic(payload: &(dyn Any + Send)) -> bool {
- matches!(panic_message(payload), Some("overflow" | "offset overflow"))
-}
-
#[cfg(test)]
mod tests {
use super::*;
- use arrow::error::ArrowError;
+ use arrow::array::{Array, ArrayDataBuilder, Int32Array, ListArray};
+ use arrow::buffer::Buffer;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_execution::memory_pool::{
+ MemoryConsumer, MemoryPool, UnboundedMemoryPool,
+ };
+
+ fn overflow_list_batch() -> RecordBatch {
+ let values_field = Arc::new(Field::new_list_field(DataType::Int32, true));
+ // SAFETY: This intentionally constructs an invalid child length so
+ // Arrow's interleave hits offset overflow before touching child data.
+ let list = ListArray::from(unsafe {
+ ArrayDataBuilder::new(DataType::List(Arc::clone(&values_field)))
+ .len(1)
+ .add_buffer(Buffer::from_slice_ref([0_i32, i32::MAX]))
+ .add_child_data(Int32Array::from(Vec::<i32>::new()).to_data())
+ .build_unchecked()
+ });
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "list_col",
+ DataType::List(values_field),
+ true,
+ )]));
+ RecordBatch::try_new(schema, vec![Arc::new(list)]).unwrap()
+ }
#[test]
fn test_retry_interleave_halves_rows_until_success() {
@@ -336,43 +323,37 @@ mod tests {
}
#[test]
- fn test_recover_offset_overflow_from_panic() {
- let error = recover_offset_overflow_from_panic(
- || -> std::result::Result<(), ArrowError> { panic!("offset overflow") },
- )
- .unwrap_err();
-
- assert!(is_offset_overflow(&error));
+ fn test_is_offset_overflow_matches_arrow_error() {
+ assert!(is_offset_overflow(&offset_overflow_error()));
}
#[test]
- fn test_recover_offset_overflow_from_panic_rethrows_unrelated_panics() {
- let panic_payload = catch_unwind(AssertUnwindSafe(|| {
- let _ = recover_offset_overflow_from_panic(
- || -> std::result::Result<(), ArrowError> { panic!("capacity overflow") },
- );
- }));
-
- assert!(panic_payload.is_err());
+ fn test_retry_interleave_does_not_retry_non_offset_errors() {
+ let mut attempts = Vec::new();
+
+ let error = retry_interleave(4, 4, |rows_to_emit| {
+ attempts.push(rows_to_emit);
+ Err::<(), _>(DataFusionError::Execution("boom".into()))
+ })
+ .unwrap_err();
+
+ assert_eq!(attempts, vec![4]);
+ assert!(matches!(error, DataFusionError::Execution(msg) if msg == "boom"));
}
#[test]
- fn test_is_arrow_offset_overflow_panic() {
- let overflow = Box::new("overflow") as Box<dyn Any + Send>;
- assert!(is_arrow_offset_overflow_panic(overflow.as_ref()));
-
- let offset_overflow =
- Box::new(String::from("offset overflow")) as Box<dyn Any + Send>;
- assert!(is_arrow_offset_overflow_panic(offset_overflow.as_ref()));
-
- let capacity_overflow = Box::new("capacity overflow") as Box<dyn Any + Send>;
- assert!(!is_arrow_offset_overflow_panic(capacity_overflow.as_ref()));
-
- let arithmetic_overflow =
- Box::new(String::from("attempt to multiply with overflow"))
- as Box<dyn Any + Send>;
- assert!(!is_arrow_offset_overflow_panic(
- arithmetic_overflow.as_ref()
- ));
+ fn test_try_interleave_columns_surfaces_arrow_offset_overflow() {
+ let batch = overflow_list_batch();
+ let schema = batch.schema();
+ let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
+ let reservation = MemoryConsumer::new("test").register(&pool);
+ let mut builder = BatchBuilder::new(schema, 1, 2, reservation);
+ builder.push_batch(0, batch).unwrap();
+
+ let error = builder
+ .try_interleave_columns(&[(0, 0), (0, 0)])
+ .unwrap_err();
+
+ assert!(is_offset_overflow(&error));
}
}