Skip to content

Apache DataFusion SQL Query Engine

fix: reject LargeList offsets that exceed i32 (#25516)

Liang-Chi Hsieh38a1865b7e18
diff --git a/datafusion/spark/src/function/map/utils.rs b/datafusion/spark/src/function/map/utils.rs
index cd8fa9ccdd..6f6fec5b3b 100644
--- a/datafusion/spark/src/function/map/utils.rs
+++ b/datafusion/spark/src/function/map/utils.rs
@@ -25,7 +25,7 @@ use arrow::array::{
use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::compute::{filter, take};
use arrow::datatypes::{DataType, Field, Fields};
-use datafusion_common::{Result, ScalarValue, exec_err};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err};
/// Helper function to get element [`DataType`]
/// from [`List`](DataType::List)/[`LargeList`](DataType::LargeList)/[`FixedSizeList`](DataType::FixedSizeList)<br>
@@ -70,8 +70,14 @@ pub fn get_list_offsets(array: &ArrayRef) -> Result<Cow<'_, [i32]>> {
.as_list::<i64>()
.offsets()
.iter()
- .map(|i| *i as i32)
- .collect::<Vec<_>>(),
+ .map(|offset| {
+ i32::try_from(*offset).map_err(|_| {
+ exec_datafusion_err!(
+ "get_list_offsets: LargeList offset {offset} cannot be represented as i32"
+ )
+ })
+ })
+ .collect::<Result<Vec<_>>>()?,
)),
DataType::FixedSizeList(_, size) => Ok(Cow::Owned(
(0..=array.len() as i32).map(|i| size * i).collect(),
@@ -259,7 +265,7 @@ fn map_deduplicate_keys(
#[cfg(test)]
mod tests {
use super::*;
- use arrow::array::{Int32Array, StringArray};
+ use arrow::array::{Int32Array, LargeListArray, NullArray, StringArray};
use arrow::datatypes::Int32Type;
fn int32_utf8_inputs(
@@ -271,6 +277,24 @@ mod tests {
(keys, values)
}
+ #[test]
+ fn large_list_offsets_do_not_truncate() {
+ let start = i64::from(i32::MAX) + 1;
+ let values: ArrayRef = Arc::new(NullArray::new((start + 1) as usize));
+ let array: ArrayRef = Arc::new(
+ LargeListArray::new(
+ Arc::new(Field::new_list_field(DataType::Null, true)),
+ OffsetBuffer::new(vec![0, start, start + 1].into()),
+ values,
+ None,
+ )
+ .slice(1, 1),
+ );
+
+ let err = get_list_offsets(&array).unwrap_err().to_string();
+ assert!(err.contains("cannot be represented as i32"), "{err}");
+ }
+
#[test]
fn happy_path_two_rows_no_duplicates() {
let (keys, values) =