Skip to content

Apache DataFusion SQL Query Engine

Support Dictionary Arrays in MIN/MAX Aggregates (#21315)

kosiew310dd5d40b75
diff --git a/datafusion/functions-aggregate-common/src/min_max.rs b/datafusion/functions-aggregate-common/src/min_max.rs
index 27620221cf..a9764d6ac6 100644
--- a/datafusion/functions-aggregate-common/src/min_max.rs
+++ b/datafusion/functions-aggregate-common/src/min_max.rs
@@ -18,8 +18,8 @@
//! Basic min/max functionality shared across DataFusion aggregate functions
use arrow::array::{
- ArrayRef, AsArray as _, BinaryArray, BinaryViewArray, BooleanArray, Date32Array,
- Date64Array, Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
+ ArrayRef, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array,
+ Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
DurationMicrosecondArray, DurationMillisecondArray, DurationNanosecondArray,
DurationSecondArray, FixedSizeBinaryArray, Float16Array, Float32Array, Float64Array,
Int8Array, Int16Array, Int32Array, Int64Array, IntervalDayTimeArray,
@@ -141,10 +141,25 @@ macro_rules! min_max_generic {
}};
}
-// min/max of two scalar values of the same type
macro_rules! min_max {
($VALUE:expr, $DELTA:expr, $OP:ident) => {{
- Ok(match ($VALUE, $DELTA) {
+ match choose_min_max!($OP) {
+ Ordering::Greater => Ok(min_max_scalar_impl!($VALUE, $DELTA, min)),
+ Ordering::Less => Ok(min_max_scalar_impl!($VALUE, $DELTA, max)),
+ Ordering::Equal => {
+ unreachable!("min/max comparisons do not use equal ordering")
+ }
+ }
+ }};
+}
+
+// min/max of two logically compatible scalar values.
+// Dictionary scalars participate by comparing their inner logical values.
+// When both inputs are dictionaries, matching key types are preserved in the
+// result; differing key types remain an unexpected invariant violation.
+macro_rules! min_max_scalar_impl {
+ ($VALUE:expr, $DELTA:expr, $OP:ident) => {{
+ match ($VALUE, $DELTA) {
(ScalarValue::Null, ScalarValue::Null) => ScalarValue::Null,
(
lhs @ ScalarValue::Decimal32(lhsv, lhsp, lhss),
@@ -413,16 +428,54 @@ macro_rules! min_max {
min_max_generic!(lhs, rhs, $OP)
}
+ (
+ ScalarValue::Dictionary(lhs_dict_key_type, lhs_dict_value),
+ ScalarValue::Dictionary(rhs_dict_key_type, rhs_dict_value),
+ ) => {
+ if lhs_dict_key_type != rhs_dict_key_type {
+ return internal_err!(
+ "MIN/MAX is not expected to receive dictionary scalars with different key types ({:?} vs {:?})",
+ lhs_dict_key_type,
+ rhs_dict_key_type
+ );
+ }
+
+ let result = min_max_scalar(
+ lhs_dict_value.as_ref(),
+ rhs_dict_value.as_ref(),
+ choose_min_max!($OP),
+ )?;
+ ScalarValue::Dictionary(lhs_dict_key_type.clone(), Box::new(result))
+ }
+ (ScalarValue::Dictionary(_, lhs_dict_value), rhs_scalar) => {
+ min_max_scalar(lhs_dict_value.as_ref(), rhs_scalar, choose_min_max!($OP))?
+ }
+ (lhs_scalar, ScalarValue::Dictionary(_, rhs_dict_value)) => {
+ min_max_scalar(lhs_scalar, rhs_dict_value.as_ref(), choose_min_max!($OP))?
+ }
+
e => {
return internal_err!(
- "MIN/MAX is not expected to receive scalars of incompatible types {:?}",
+ "MIN/MAX is not expected to receive logically incompatible scalar values {:?}",
e
)
}
- })
+ }
}};
}
+fn min_max_scalar(
+ lhs: &ScalarValue,
+ rhs: &ScalarValue,
+ ordering: Ordering,
+) -> Result<ScalarValue> {
+ match ordering {
+ Ordering::Greater => Ok(min_max_scalar_impl!(lhs, rhs, min)),
+ Ordering::Less => Ok(min_max_scalar_impl!(lhs, rhs, max)),
+ Ordering::Equal => unreachable!("min/max comparisons do not use equal ordering"),
+ }
+}
+
/// An accumulator to compute the maximum value
#[derive(Debug, Clone)]
pub struct MaxAccumulator {
@@ -760,16 +813,11 @@ pub fn min_batch(values: &ArrayRef) -> Result<ScalarValue> {
min_binary_view
)
}
- DataType::Struct(_) => min_max_batch_generic(values, Ordering::Greater)?,
- DataType::List(_) => min_max_batch_generic(values, Ordering::Greater)?,
- DataType::LargeList(_) => min_max_batch_generic(values, Ordering::Greater)?,
- DataType::FixedSizeList(_, _) => {
- min_max_batch_generic(values, Ordering::Greater)?
- }
- DataType::Dictionary(_, _) => {
- let values = values.as_any_dictionary().values();
- min_batch(values)?
- }
+ DataType::Struct(_)
+ | DataType::List(_)
+ | DataType::LargeList(_)
+ | DataType::FixedSizeList(_, _)
+ | DataType::Dictionary(_, _) => min_max_batch_generic(values, Ordering::Greater)?,
_ => min_max_batch!(values, min),
})
}
@@ -843,14 +891,94 @@ pub fn max_batch(values: &ArrayRef) -> Result<ScalarValue> {
let value = value.map(|e| e.to_vec());
ScalarValue::FixedSizeBinary(*size, value)
}
- DataType::Struct(_) => min_max_batch_generic(values, Ordering::Less)?,
- DataType::List(_) => min_max_batch_generic(values, Ordering::Less)?,
- DataType::LargeList(_) => min_max_batch_generic(values, Ordering::Less)?,
- DataType::FixedSizeList(_, _) => min_max_batch_generic(values, Ordering::Less)?,
- DataType::Dictionary(_, _) => {
- let values = values.as_any_dictionary().values();
- max_batch(values)?
- }
+ DataType::Struct(_)
+ | DataType::List(_)
+ | DataType::LargeList(_)
+ | DataType::FixedSizeList(_, _)
+ | DataType::Dictionary(_, _) => min_max_batch_generic(values, Ordering::Less)?,
_ => min_max_batch!(values, max),
})
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn min_max_dictionary_and_scalar_compare_by_inner_value() -> Result<()> {
+ let dictionary = ScalarValue::Dictionary(
+ Box::new(DataType::Int32),
+ Box::new(ScalarValue::Float32(Some(1.0))),
+ );
+ let scalar = ScalarValue::Float32(Some(2.0));
+
+ let result = min_max_scalar(&dictionary, &scalar, Ordering::Less)?;
+
+ assert_eq!(result, ScalarValue::Float32(Some(2.0)));
+ Ok(())
+ }
+
+ #[test]
+ fn min_max_dictionary_same_key_type_rewraps_result() -> Result<()> {
+ let lhs = ScalarValue::Dictionary(
+ Box::new(DataType::Int32),
+ Box::new(ScalarValue::Float32(Some(1.0))),
+ );
+ let rhs = ScalarValue::Dictionary(
+ Box::new(DataType::Int32),
+ Box::new(ScalarValue::Float32(Some(2.0))),
+ );
+
+ let result = min_max_scalar(&lhs, &rhs, Ordering::Less)?;
+
+ assert_eq!(
+ result,
+ ScalarValue::Dictionary(
+ Box::new(DataType::Int32),
+ Box::new(ScalarValue::Float32(Some(2.0))),
+ )
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn min_max_dictionary_different_key_types_error() -> Result<()> {
+ let lhs = ScalarValue::Dictionary(
+ Box::new(DataType::Int8),
+ Box::new(ScalarValue::Float32(Some(1.0))),
+ );
+ let rhs = ScalarValue::Dictionary(
+ Box::new(DataType::Int32),
+ Box::new(ScalarValue::Float32(Some(2.0))),
+ );
+
+ let error: DataFusionError =
+ min_max_scalar(&lhs, &rhs, Ordering::Less).unwrap_err();
+
+ assert!(
+ error
+ .to_string()
+ .contains("dictionary scalars with different key types")
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn min_max_dictionary_and_incompatible_scalar_error() -> Result<()> {
+ let dictionary = ScalarValue::Dictionary(
+ Box::new(DataType::Int32),
+ Box::new(ScalarValue::Float32(Some(1.0))),
+ );
+ let scalar = ScalarValue::Int32(Some(2));
+
+ let error: DataFusionError =
+ min_max_scalar(&dictionary, &scalar, Ordering::Less).unwrap_err();
+
+ assert!(
+ error
+ .to_string()
+ .contains("logically incompatible scalar values")
+ );
+ Ok(())
+ }
+}
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs
index 939ee2a2d8..f4eaaab853 100644
--- a/datafusion/functions-aggregate/src/min_max.rs
+++ b/datafusion/functions-aggregate/src/min_max.rs
@@ -1003,12 +1003,13 @@ mod tests {
use super::*;
use arrow::{
array::{
- DictionaryArray, Float32Array, Int32Array, IntervalDayTimeArray,
- IntervalMonthDayNanoArray, IntervalYearMonthArray, StringArray,
+ Array, DictionaryArray, Float32Array, Int8Array, Int32Array,
+ IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray,
+ PrimitiveArray, StringArray,
},
datatypes::{
- IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit,
- IntervalYearMonthType,
+ ArrowDictionaryKeyType, IntervalDayTimeType, IntervalMonthDayNanoType,
+ IntervalUnit, IntervalYearMonthType,
},
};
use std::sync::Arc;
@@ -1258,7 +1259,178 @@ mod tests {
let mut max_acc = MaxAccumulator::try_new(&rt_type)?;
max_acc.update_batch(&[Arc::clone(&dict_array_ref)])?;
let max_result = max_acc.evaluate()?;
- assert_eq!(max_result, ScalarValue::Utf8(Some("🦀".to_string())));
+ assert_eq!(max_result, ScalarValue::Utf8(Some("d".to_string())));
+ Ok(())
+ }
+
+ fn dict_scalar(key_type: DataType, inner: ScalarValue) -> ScalarValue {
+ ScalarValue::Dictionary(Box::new(key_type), Box::new(inner))
+ }
+
+ fn utf8_dict_scalar(key_type: DataType, value: &str) -> ScalarValue {
+ dict_scalar(key_type, ScalarValue::Utf8(Some(value.to_string())))
+ }
+
+ fn string_dictionary_batch(values: &[&str], keys: &[Option<i32>]) -> ArrayRef {
+ string_dictionary_batch_with_keys(Int32Array::from(keys.to_vec()), values)
+ }
+
+ fn string_dictionary_batch_with_keys<K>(
+ keys: PrimitiveArray<K>,
+ values: &[&str],
+ ) -> ArrayRef
+ where
+ K: ArrowDictionaryKeyType,
+ {
+ let values = Arc::new(StringArray::from(values.to_vec())) as ArrayRef;
+ Arc::new(DictionaryArray::try_new(keys, values).unwrap()) as ArrayRef
+ }
+
+ fn optional_string_dictionary_batch(
+ values: &[Option<&str>],
+ keys: &[Option<i32>],
+ ) -> ArrayRef {
+ let values = Arc::new(StringArray::from(values.to_vec())) as ArrayRef;
+ Arc::new(
+ DictionaryArray::try_new(Int32Array::from(keys.to_vec()), values).unwrap(),
+ ) as ArrayRef
+ }
+
+ fn float_dictionary_batch(values: &[f32], keys: &[Option<i32>]) -> ArrayRef {
+ let values = Arc::new(Float32Array::from(values.to_vec())) as ArrayRef;
+ Arc::new(
+ DictionaryArray::try_new(Int32Array::from(keys.to_vec()), values).unwrap(),
+ ) as ArrayRef
+ }
+
+ fn evaluate_dictionary_accumulator(
+ mut acc: impl Accumulator,
+ batches: &[ArrayRef],
+ ) -> Result<ScalarValue> {
+ for batch in batches {
+ acc.update_batch(&[Arc::clone(batch)])?;
+ }
+ acc.evaluate()
+ }
+
+ fn assert_dictionary_min_max(
+ dict_type: &DataType,
+ batches: &[ArrayRef],
+ expected_min: &str,
+ expected_max: &str,
+ ) -> Result<()> {
+ let key_type = match dict_type {
+ DataType::Dictionary(key_type, _) => key_type.as_ref().clone(),
+ other => panic!("expected dictionary type, got {other:?}"),
+ };
+
+ let min_result = evaluate_dictionary_accumulator(
+ MinAccumulator::try_new(dict_type)?,
+ batches,
+ )?;
+ assert_eq!(min_result, utf8_dict_scalar(key_type.clone(), expected_min));
+
+ let max_result = evaluate_dictionary_accumulator(
+ MaxAccumulator::try_new(dict_type)?,
+ batches,
+ )?;
+ assert_eq!(max_result, utf8_dict_scalar(key_type, expected_max));
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_min_max_dictionary_without_coercion() -> Result<()> {
+ let dict_array_ref = string_dictionary_batch(
+ &["b", "c", "a", "d"],
+ &[Some(0), Some(1), Some(2), Some(3)],
+ );
+ let dict_type = dict_array_ref.data_type().clone();
+
+ assert_dictionary_min_max(&dict_type, &[dict_array_ref], "a", "d")
+ }
+
+ #[test]
+ fn test_min_max_dictionary_with_nulls() -> Result<()> {
+ let dict_array_ref = string_dictionary_batch(
+ &["b", "c", "a"],
+ &[None, Some(0), None, Some(1), Some(2)],
+ );
+ let dict_type = dict_array_ref.data_type().clone();
+
+ assert_dictionary_min_max(&dict_type, &[dict_array_ref], "a", "c")
+ }
+
+ #[test]
+ fn test_min_max_dictionary_ignores_unreferenced_values() -> Result<()> {
+ let dict_array_ref =
+ string_dictionary_batch(&["a", "z", "zz_unused"], &[Some(1), Some(1), None]);
+ let dict_type = dict_array_ref.data_type().clone();
+
+ assert_dictionary_min_max(&dict_type, &[dict_array_ref], "z", "z")
+ }
+
+ #[test]
+ fn test_min_max_dictionary_ignores_referenced_null_values() -> Result<()> {
+ let dict_array_ref = optional_string_dictionary_batch(
+ &[Some("b"), None, Some("a"), Some("d")],
+ &[Some(0), Some(1), Some(2), Some(3)],
+ );
+ let dict_type = dict_array_ref.data_type().clone();
+
+ assert_dictionary_min_max(&dict_type, &[dict_array_ref], "a", "d")
+ }
+
+ #[test]
+ fn test_min_max_dictionary_multi_batch() -> Result<()> {
+ let dict_type =
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+ let batch1 = string_dictionary_batch(&["b", "c"], &[Some(0), Some(1)]);
+ let batch2 = string_dictionary_batch(&["a", "d"], &[Some(0), Some(1)]);
+
+ assert_dictionary_min_max(&dict_type, &[batch1, batch2], "a", "d")
+ }
+
+ #[test]
+ fn test_min_max_dictionary_int8_keys() -> Result<()> {
+ let dict_type =
+ DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8));
+ let dict_array_ref = string_dictionary_batch_with_keys(
+ Int8Array::from(vec![Some(0), Some(1), Some(2), Some(3)]),
+ &["b", "c", "a", "d"],
+ );
+
+ assert_dictionary_min_max(&dict_type, &[dict_array_ref], "a", "d")
+ }
+
+ #[test]
+ fn test_min_max_dictionary_float_with_nans() -> Result<()> {
+ let dict_type =
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Float32));
+ let batch1 = float_dictionary_batch(&[0.0, f32::NAN], &[Some(0), Some(1)]);
+ let batch2 = float_dictionary_batch(&[f32::NEG_INFINITY], &[Some(0)]);
+
+ let min_result = evaluate_dictionary_accumulator(
+ MinAccumulator::try_new(&dict_type)?,
+ &[Arc::clone(&batch1), Arc::clone(&batch2)],
+ )?;
+ assert_eq!(
+ min_result,
+ dict_scalar(
+ DataType::Int32,
+ ScalarValue::Float32(Some(f32::NEG_INFINITY)),
+ )
+ );
+
+ let max_result = evaluate_dictionary_accumulator(
+ MaxAccumulator::try_new(&dict_type)?,
+ &[batch1, batch2],
+ )?;
+ assert_eq!(
+ max_result,
+ dict_scalar(DataType::Int32, ScalarValue::Float32(Some(f32::NAN)))
+ );
+
Ok(())
}
}