Skip to content

Apache DataFusion SQL Query Engine

feat(physical-plan): generic Rows-backed GroupColumn keeps mixed schemas on the column-wise path (#23523)

Qi Zhu68d587468073
diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
index f275d777c3..5163948bd5 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
@@ -21,6 +21,7 @@ mod boolean;
mod bytes;
pub mod bytes_view;
pub mod primitive;
+pub mod row_backed;
use std::mem::{self, size_of};
@@ -28,6 +29,7 @@ use crate::aggregates::group_values::GroupValues;
use crate::aggregates::group_values::multi_group_by::{
boolean::BooleanGroupValueBuilder, bytes::ByteGroupValueBuilder,
bytes_view::ByteViewGroupValueBuilder, primitive::PrimitiveGroupValueBuilder,
+ row_backed::RowsGroupColumn,
};
use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};
use arrow::compute::cast;
@@ -923,6 +925,15 @@ macro_rules! instantiate_primitive {
/// builder for. The `group_column_supported_type_matches_make_group_column`
/// test below pins this biconditional.
fn group_column_supported_type(data_type: &DataType) -> bool {
+ // Nested types (Struct / List / LargeList / FixedSizeList, recursively) have
+ // no type-specialized `GroupColumn`; they are handled by the generic
+ // row-backed fallback in `make_group_column` whenever arrow's row format can
+ // encode them. Gate the fallback to nested types so intentionally-excluded
+ // scalar types (e.g. Float16, Decimal256) stay on `GroupValuesRows` and the
+ // `group_column_supported_type` ⇔ `make_group_column` invariant holds.
+ if data_type.is_nested() {
+ return RowsGroupColumn::supports_type(data_type);
+ }
matches!(
*data_type,
DataType::Int8
@@ -1067,6 +1078,14 @@ fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {
v.push(Box::new(BooleanGroupValueBuilder::<false>::new()));
}
}
+ // Generic fallback for nested types (Struct / List / LargeList /
+ // FixedSizeList, recursively) that lack a type-specialized builder but
+ // can be encoded by arrow's row format. This is what lets a mixed
+ // schema keep the column-wise fast path for its native columns instead
+ // of dropping the whole key onto `GroupValuesRows`.
+ ref dt if dt.is_nested() && RowsGroupColumn::supports_type(dt) => {
+ v.push(Box::new(RowsGroupColumn::try_new(dt.clone())?));
+ }
_ => return not_impl_err!("{data_type} not supported in GroupValuesColumn"),
}
debug_assert_eq!(
@@ -1273,6 +1292,255 @@ mod tests {
GroupIndexView, group_column_supported_type, make_group_column, supported_schema,
};
+ /// A mixed group-by key of several native columns plus one nested column
+ /// that has no type-specialized `GroupColumn`.
+ ///
+ /// Before the generic row-backed fallback, `supported_schema` returned
+ /// `false` for this schema, so the *entire* key dropped to the row-wise
+ /// `GroupValuesRows`. Now only the nested column pays the row-encoding
+ /// cost; the native columns keep their compact column-wise storage. This
+ /// test proves both that (a) the results are identical and (b) the
+ /// column-wise path now uses less memory than the all-rows fallback.
+ #[test]
+ fn mixed_schema_column_path_uses_less_memory_than_rows_fallback() {
+ use crate::aggregates::group_values::GroupValuesRows;
+ use arrow::array::{FixedSizeListArray, Int64Array};
+ use arrow::datatypes::Int64Type;
+
+ // 8 native Int64 columns + 1 FixedSizeList<Int64, 4> ("embedding").
+ let fsl_field = Arc::new(Field::new("item", DataType::Int64, true));
+ let mut fields: Vec<Field> = (0..8)
+ .map(|i| Field::new(format!("k{i}"), DataType::Int64, false))
+ .collect();
+ fields.push(Field::new(
+ "emb",
+ DataType::FixedSizeList(Arc::clone(&fsl_field), 4),
+ true,
+ ));
+ let schema: SchemaRef = Arc::new(Schema::new(fields));
+
+ // The whole schema must now be eligible for the column-wise path.
+ assert!(
+ supported_schema(schema.as_ref()),
+ "mixed native + nested schema should be column-supported now"
+ );
+
+ // Build `n_groups` distinct rows (each row is its own group).
+ let n_groups = 4000usize;
+ let mut cols: Vec<ArrayRef> = (0..8)
+ .map(|c| {
+ let vals: Vec<i64> =
+ (0..n_groups).map(|r| (r as i64) * 8 + c as i64).collect();
+ Arc::new(Int64Array::from(vals)) as ArrayRef
+ })
+ .collect();
+ let emb: Vec<Option<Vec<Option<i64>>>> = (0..n_groups)
+ .map(|r| {
+ Some(vec![
+ Some(r as i64),
+ Some(r as i64 + 1),
+ Some(r as i64 + 2),
+ Some(r as i64 + 3),
+ ])
+ })
+ .collect();
+ cols.push(
+ Arc::new(FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(
+ emb, 4,
+ )) as ArrayRef,
+ );
+
+ // Intern the same data into both implementations.
+ let mut column_path = GroupValuesColumn::<false>::try_new(Arc::clone(&schema))
+ .expect("column path");
+ let mut rows_path =
+ GroupValuesRows::try_new(Arc::clone(&schema)).expect("rows path");
+
+ let mut g1 = vec![];
+ let mut g2 = vec![];
+ column_path.intern(&cols, &mut g1).unwrap();
+ rows_path.intern(&cols, &mut g2).unwrap();
+
+ // (a) Correctness: same number of groups and identical group assignment.
+ assert_eq!(column_path.len(), n_groups);
+ assert_eq!(rows_path.len(), n_groups);
+ assert_eq!(g1, g2, "group assignment must match the rows fallback");
+
+ // (b) Memory: the column-wise path stores the 8 native columns compactly
+ // and only row-encodes the nested one, so it should be smaller than
+ // encoding every column into rows.
+ //
+ // The delta is only printed here — a hard `column_size < rows_size`
+ // assert would be brittle to future Arrow row-format or memory-
+ // accounting changes without reflecting a grouping-correctness
+ // regression. Track the memory improvement via benchmarks instead.
+ let column_size = column_path.size();
+ let rows_size = rows_path.size();
+ println!(
+ "mixed-schema group values size: column-wise = {column_size} bytes, \
+ all-rows fallback = {rows_size} bytes \
+ ({:.1}% of fallback)",
+ 100.0 * column_size as f64 / rows_size as f64
+ );
+
+ // Emitted values must be equal too (compare via the rows fallback which
+ // is the established reference implementation).
+ let out_col = column_path.emit(EmitTo::All).unwrap();
+ let out_row = rows_path.emit(EmitTo::All).unwrap();
+ assert_eq!(out_col.len(), out_row.len());
+ for (a, b) in out_col.iter().zip(out_row.iter()) {
+ assert_eq!(a.as_ref(), b.as_ref());
+ }
+ }
+
+ /// Relabel a group-index vector so labels are assigned in order of first
+ /// appearance. Two vectors are equivalent groupings iff their canonical
+ /// forms are equal — this ignores the (opaque, non-semantic) difference in
+ /// group-index numbering between the vectorized column path and the
+ /// sequential rows fallback.
+ ///
+ /// The [`GroupValues`] trait only guarantees that equal keys receive the
+ /// same group-id and that new keys receive a fresh id; the order in which
+ /// new ids are handed out is deliberately not part of the contract, and
+ /// can differ between correct implementations (e.g. because of internal
+ /// hash-map ordering). Canonicalizing before comparison is what lets us
+ /// assert equivalence across implementations.
+ fn canonical_grouping(groups: &[usize]) -> Vec<usize> {
+ let mut map = HashMap::new();
+ let mut next = 0usize;
+ groups
+ .iter()
+ .map(|&g| {
+ *map.entry(g).or_insert_with(|| {
+ let v = next;
+ next += 1;
+ v
+ })
+ })
+ .collect()
+ }
+
+ /// The generic row-backed column must be behavior-preserving: for the
+ /// nested columns it now handles, `GroupValuesColumn` must induce the same
+ /// grouping (partition of rows) as the established `GroupValuesRows`
+ /// fallback — including the float `-0.0` / `+0.0` / `NaN` edge cases decided
+ /// jointly by hashing and the row format.
+ #[test]
+ fn nested_float_edge_cases_match_rows_fallback() {
+ use crate::aggregates::group_values::GroupValuesRows;
+ use arrow::array::{FixedSizeListArray, Float64Array};
+
+ let item = Arc::new(Field::new("item", DataType::Float64, true));
+ let schema: SchemaRef = Arc::new(Schema::new(vec![Field::new(
+ "emb",
+ DataType::FixedSizeList(Arc::clone(&item), 2),
+ true,
+ )]));
+ assert!(supported_schema(schema.as_ref()));
+
+ // Rows exercising +0.0 vs -0.0, two NaN bit patterns, and inner nulls.
+ let nan = f64::NAN;
+ let other_nan = f64::from_bits(0x7ff8_0000_0000_0001);
+ let values = Float64Array::from(vec![
+ Some(0.0),
+ Some(1.0), // [ +0.0, 1.0 ]
+ Some(-0.0),
+ Some(1.0), // [ -0.0, 1.0 ]
+ Some(nan),
+ Some(2.0), // [ NaN, 2.0 ]
+ Some(other_nan),
+ Some(2.0), // [ NaN', 2.0 ]
+ Some(0.0),
+ Some(1.0), // [ +0.0, 1.0 ] (dup of row 0)
+ ]);
+ let field_ref = Arc::new(Field::new("item", DataType::Float64, true));
+ let input: ArrayRef = Arc::new(FixedSizeListArray::new(
+ field_ref,
+ 2,
+ Arc::new(values),
+ None,
+ ));
+
+ let cols = vec![input];
+
+ let mut column_path =
+ GroupValuesColumn::<false>::try_new(Arc::clone(&schema)).unwrap();
+ let mut rows_path = GroupValuesRows::try_new(Arc::clone(&schema)).unwrap();
+
+ let mut g1 = vec![];
+ let mut g2 = vec![];
+ column_path.intern(&cols, &mut g1).unwrap();
+ rows_path.intern(&cols, &mut g2).unwrap();
+
+ assert_eq!(
+ canonical_grouping(&g1),
+ canonical_grouping(&g2),
+ "column-wise path must induce the same grouping as the rows fallback \
+ on float edge cases (got column={g1:?}, rows={g2:?})"
+ );
+ assert_eq!(column_path.len(), rows_path.len());
+ }
+
+ /// Equivalence across multiple `intern` batches and `EmitTo::First(n)`.
+ #[test]
+ fn multi_batch_and_emit_first_matches_rows_fallback() {
+ use crate::aggregates::group_values::GroupValuesRows;
+ use arrow::array::{FixedSizeListArray, Int32Array};
+ use arrow::datatypes::Int32Type;
+
+ let item = Arc::new(Field::new("item", DataType::Int32, true));
+ let schema: SchemaRef = Arc::new(Schema::new(vec![
+ Field::new("k", DataType::Int32, false),
+ Field::new("emb", DataType::FixedSizeList(Arc::clone(&item), 2), true),
+ ]));
+
+ let make_batch = |base: i32| -> Vec<ArrayRef> {
+ let k = Arc::new(Int32Array::from(vec![base, base + 1, base])) as ArrayRef;
+ let emb: Vec<Option<Vec<Option<i32>>>> = vec![
+ Some(vec![Some(base), Some(base)]),
+ Some(vec![Some(base + 1), None]),
+ Some(vec![Some(base), Some(base)]), // dup of row 0
+ ];
+ let emb = Arc::new(
+ FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(emb, 2),
+ ) as ArrayRef;
+ vec![k, emb]
+ };
+
+ let mut column_path =
+ GroupValuesColumn::<false>::try_new(Arc::clone(&schema)).unwrap();
+ let mut rows_path = GroupValuesRows::try_new(Arc::clone(&schema)).unwrap();
+
+ for base in [0, 10, 0] {
+ let cols = make_batch(base);
+ let (mut a, mut b) = (vec![], vec![]);
+ column_path.intern(&cols, &mut a).unwrap();
+ rows_path.intern(&cols, &mut b).unwrap();
+ // Same grouping (partition), even if the opaque group-index labels
+ // differ between the vectorized and sequential paths.
+ assert_eq!(
+ canonical_grouping(&a),
+ canonical_grouping(&b),
+ "grouping must match for batch base={base}"
+ );
+ }
+
+ let total_groups = column_path.len();
+ assert_eq!(total_groups, rows_path.len());
+
+ // `EmitTo::First(n)` then `EmitTo::All` on the nested column path must
+ // work and together emit exactly `total_groups` rows. (Cross-path value
+ // equality is covered by `mixed_schema_...` and the row_backed unit
+ // tests; group-index ordering differs here so we check counts.)
+ let col_first = column_path.emit(EmitTo::First(2)).unwrap();
+ assert_eq!(col_first[0].len(), 2);
+ let col_rest = column_path.emit(EmitTo::All).unwrap();
+ assert_eq!(col_first[0].len() + col_rest[0].len(), total_groups);
+ // Column count / schema preserved on both emits.
+ assert_eq!(col_first.len(), schema.fields().len());
+ assert_eq!(col_rest.len(), schema.fields().len());
+ }
+
/// CRITICAL invariant: if `group_column_supported_type(t)` returns true
/// the dispatcher must accept that type at intern time, and conversely
/// if `group_column_supported_type(t)` returns false the planner must
diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs
new file mode 100644
index 0000000000..29beb3bd66
--- /dev/null
+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs
@@ -0,0 +1,1014 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! A generic [`GroupColumn`] backed by the arrow row format.
+//!
+//! Unlike the type-specialized builders in this module (primitive, byte,
+//! boolean, ...), [`RowsGroupColumn`] works for *any* data type that arrow's
+//! [`RowConverter`] can encode — including nested types such as `Struct`,
+//! `List`, `LargeList` and `FixedSizeList`. It stores one group value per row
+//! in a single-column [`Rows`] buffer and compares group keys by their encoded
+//! bytes.
+//!
+//! # Why this exists
+//!
+//! [`GroupValuesColumn`] can only be used when *every* column of the group-by
+//! key has a [`GroupColumn`] implementation; otherwise the whole aggregation
+//! falls back to the row-wise [`GroupValuesRows`], which is materially slower
+//! and heavier for the columns that *would* have qualified for the column-wise
+//! fast path. By providing a generic fallback `GroupColumn`, a schema like
+//! `GROUP BY int_col, struct_col` keeps `int_col` on its fast native builder
+//! and only pays the row-encoding cost on `struct_col`, instead of dragging both
+//! columns onto `GroupValuesRows`.
+//!
+//! # Relationship to hashing
+//!
+//! This column does not hash anything itself: [`GroupValuesColumn`] hashes the
+//! raw input columns via `create_hashes`, which already supports nested types.
+//! Equality is decided here by comparing arrow-row bytes. For the two to agree
+//! on group identity, values that this column considers equal must hash equal —
+//! see the float `-0.0` / `NaN` note on [`RowsGroupColumn`].
+//!
+//! [`GroupValuesColumn`]: crate::aggregates::group_values::multi_group_by::GroupValuesColumn
+//! [`GroupValuesRows`]: crate::aggregates::group_values::GroupValuesRows
+
+use crate::aggregates::group_values::multi_group_by::GroupColumn;
+use crate::aggregates::group_values::row::encode_array_if_necessary;
+
+use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};
+use arrow::datatypes::DataType;
+use arrow::row::{RowConverter, Rows, SortField};
+use datafusion_common::{DataFusionError, Result};
+
+/// A [`GroupColumn`] that stores group values for a single column in the arrow
+/// [row format], backed by a single-field [`RowConverter`].
+///
+/// # NULL semantics
+///
+/// The [`GroupColumn`] contract treats two NULLs as equal. The row format
+/// encodes NULL with a distinct sentinel, so `null`-row bytes compare equal to
+/// each other and unequal to any non-null row — matching the contract without
+/// special-casing.
+///
+/// # Float `-0.0` / `NaN`
+///
+/// Equality here is byte equality under arrow's IEEE-754 *totalOrder* row
+/// encoding, which treats `-0.0` and `+0.0` as distinct and canonicalizes
+/// `NaN`. Because hashing is performed separately (on the raw input array), a
+/// caller must ensure the two agree — e.g. by normalizing `-0.0 → +0.0` on the
+/// input columns before hashing when a float leaf is present (as
+/// [`GroupValuesRows`] does). See the module docs.
+///
+/// [row format]: arrow::row
+/// [`GroupValuesRows`]: crate::aggregates::group_values::GroupValuesRows
+pub struct RowsGroupColumn {
+ /// Single-field row converter for this column's data type.
+ row_converter: RowConverter,
+ /// Accumulated group values in row format; `group_values.row(i)` is the
+ /// group value for group index `i`.
+ group_values: Rows,
+ /// The column's expected output type. The row format decodes dictionary /
+ /// run-end encoded values to their plain value type, so emitted arrays are
+ /// re-encoded to this type in `build` / `take_n` (mirroring
+ /// `GroupValuesRows::emit`).
+ output_type: DataType,
+}
+
+/// Walk `data_type`'s subtree and return `true` if it contains a
+/// [`DataType::FixedSizeList`] whose descendant tree includes any
+/// [`DataType::Dictionary`].
+///
+/// Two-state recursion: once we cross a `FixedSizeList`, `inside_fsl`
+/// stays true for every descendant, so a `Dictionary` anywhere below
+/// counts. Above that boundary, encountering a `Dictionary` is fine —
+/// only nested containers propagate the risk.
+///
+/// TODO: this guard works around <https://github.com/apache/arrow-rs/issues/10413>
+/// (`decode_fixed_size_list` panics instead of applying the
+/// dictionary-flatten `corrected_type` step). Fixed upstream by
+/// <https://github.com/apache/arrow-rs/pull/10414> (merged 2026-07-24, not
+/// yet in a release as of arrow 59.1.0). Once DataFusion upgrades to an
+/// arrow release containing that fix, `FixedSizeList<Dictionary>` will
+/// decode like the other list-likes (flattened child, re-encoded by
+/// `encode_array_if_necessary`'s existing `FixedSizeList` arm) — remove
+/// this guard and its `supports_type` rejection at that point.
+fn contains_fsl_with_dictionary(data_type: &DataType) -> bool {
+ fn walk(dt: &DataType, inside_fsl: bool) -> bool {
+ match dt {
+ DataType::Dictionary(_, _) => inside_fsl,
+ DataType::FixedSizeList(f, _) => walk(f.data_type(), true),
+ DataType::List(f)
+ | DataType::LargeList(f)
+ | DataType::ListView(f)
+ | DataType::LargeListView(f) => walk(f.data_type(), inside_fsl),
+ DataType::Map(f, _) => walk(f.data_type(), inside_fsl),
+ DataType::Struct(fs) => fs.iter().any(|f| walk(f.data_type(), inside_fsl)),
+ DataType::RunEndEncoded(_, values) => walk(values.data_type(), inside_fsl),
+ DataType::Union(fs, _) => {
+ fs.iter().any(|(_, f)| walk(f.data_type(), inside_fsl))
+ }
+ _ => false,
+ }
+ }
+ walk(data_type, false)
+}
+
+/// Return `true` if `data_type` contains a [`DataType::Union`] or
+/// [`DataType::RunEndEncoded`] anywhere in its subtree.
+///
+/// These two nested variants can round-trip through `RowConverter` in
+/// principle, but their arrow-row decoders have not been validated by
+/// this crate's test matrix against the full range of leaf types (dict,
+/// nested, etc.). Before this PR both were handled by `GroupValuesRows`
+/// (they were not `is_nested`-eligible for `GroupValuesColumn`), so
+/// reject them here to preserve the pre-PR routing rather than route
+/// untested shapes through `RowsGroupColumn`. When we grow explicit
+/// round-trip tests for these types, this blacklist can be removed.
+fn contains_union_or_run_end_encoded(data_type: &DataType) -> bool {
+ match data_type {
+ DataType::Union(_, _) | DataType::RunEndEncoded(_, _) => true,
+ DataType::List(f)
+ | DataType::LargeList(f)
+ | DataType::ListView(f)
+ | DataType::LargeListView(f)
+ | DataType::FixedSizeList(f, _) => {
+ contains_union_or_run_end_encoded(f.data_type())
+ }
+ DataType::Map(f, _) => contains_union_or_run_end_encoded(f.data_type()),
+ DataType::Struct(fs) => fs
+ .iter()
+ .any(|f| contains_union_or_run_end_encoded(f.data_type())),
+ _ => false,
+ }
+}
+
+impl RowsGroupColumn {
+ /// Returns whether `data_type` can be handled by this generic column.
+ ///
+ /// This is stricter than [`RowConverter::supports_fields`]: the row
+ /// format also has to survive the `build` / `take_n` reverse trip
+ /// through [`RowConverter::convert_rows`], and arrow's
+ /// `decode_fixed_size_list` (arrow-row 59.1.0) skips the
+ /// dictionary-flatten correction that the other list-like decoders
+ /// apply, so any `FixedSizeList` containing a `Dictionary` leaf
+ /// panics on emit with `"FixedSizeListArray expected data type
+ /// Dictionary(...) got <flattened> for \"item\""`.
+ ///
+ /// Reject those shapes here so `make_group_column` falls back to
+ /// `GroupValuesRows`. The other list-likes (`List`, `LargeList`,
+ /// `ListView`, `LargeListView`, `Map`) do carry the correction, so
+ /// they decode without panicking — but the correction *flattens* any
+ /// dictionary child to its value type, so `build` / `take_n` must
+ /// re-encode the emitted array back to `output_type` via
+ /// `encode_array_if_necessary` (which has a reconstruction arm for
+ /// each of these containers).
+ ///
+ /// Additionally, `Union` and `RunEndEncoded` are rejected because
+ /// they were routed to `GroupValuesRows` before this column existed
+ /// and their arrow-row round-trip has not been covered by this
+ /// crate's tests yet. Keeping them on the pre-PR path avoids
+ /// introducing an untested code path for those types.
+ pub fn supports_type(data_type: &DataType) -> bool {
+ if contains_fsl_with_dictionary(data_type) {
+ return false;
+ }
+ if contains_union_or_run_end_encoded(data_type) {
+ return false;
+ }
+ RowConverter::supports_fields(&[SortField::new(data_type.clone())])
+ }
+
+ /// Create an empty [`RowsGroupColumn`] for `data_type`.
+ pub fn try_new(data_type: DataType) -> Result<Self> {
+ let row_converter = RowConverter::new(vec![SortField::new(data_type.clone())])?;
+ let group_values = row_converter.empty_rows(0, 0);
+ Ok(Self {
+ row_converter,
+ group_values,
+ output_type: data_type,
+ })
+ }
+
+ /// Materialize `rows` into a single array of `self.output_type`, re-applying
+ /// dictionary / run-end encoding the row format strips on decode.
+ fn rows_to_array<'a>(
+ &self,
+ rows: impl IntoIterator<Item = arrow::row::Row<'a>>,
+ ) -> ArrayRef {
+ let mut arrays = self
+ .row_converter
+ .convert_rows(rows)
+ .expect("row conversion during emit");
+ debug_assert_eq!(arrays.len(), 1, "single-field row converter");
+ let array = arrays.swap_remove(0);
+ encode_array_if_necessary(&array, &self.output_type)
+ .expect("dictionary re-encode during emit")
+ }
+
+ /// Encode a whole incoming column into the row format.
+ fn convert(&self, array: &ArrayRef) -> Result<Rows> {
+ self.row_converter
+ .convert_columns(std::slice::from_ref(array))
+ .map_err(DataFusionError::from)
+ }
+}
+
+impl GroupColumn for RowsGroupColumn {
+ fn equal_to(&self, lhs_row: usize, array: &ArrayRef, rhs_row: usize) -> bool {
+ // Scalar path (hash-collision remainder / streaming). Encode just the
+ // single incoming row rather than the whole column. The vectorized
+ // methods below encode the batch once; this path is expected to be rare.
+ let incoming = self
+ .convert(&array.slice(rhs_row, 1))
+ .expect("row conversion during equal_to");
+ self.group_values.row(lhs_row) == incoming.row(0)
+ }
+
+ fn append_val(&mut self, array: &ArrayRef, row: usize) -> Result<()> {
+ let incoming = self.convert(&array.slice(row, 1))?;
+ self.group_values.push(incoming.row(0));
+ Ok(())
+ }
+
+ fn vectorized_equal_to(
+ &self,
+ lhs_rows: &[usize],
+ array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder,
+ ) {
+ // Encode the incoming column once for the whole batch.
+ let incoming = self
+ .convert(array)
+ .expect("row conversion during vectorized_equal_to");
+ for (idx, (&lhs_row, &rhs_row)) in
+ lhs_rows.iter().zip(rhs_rows.iter()).enumerate()
+ {
+ // Preserve the AND-accumulate contract: skip rows already false.
+ if !equal_to_results.get_bit(idx) {
+ continue;
+ }
+ if self.group_values.row(lhs_row) != incoming.row(rhs_row) {
+ equal_to_results.set_bit(idx, false);
+ }
+ }
+ }
+
+ fn vectorized_append(&mut self, array: &ArrayRef, rows: &[usize]) -> Result<()> {
+ // Encode the incoming column once, then push the selected rows.
+ let incoming = self.convert(array)?;
+ for &row in rows {
+ self.group_values.push(incoming.row(row));
+ }
+ Ok(())
+ }
+
+ fn len(&self) -> usize {
+ self.group_values.num_rows()
+ }
+
+ fn size(&self) -> usize {
+ self.row_converter.size() + self.group_values.size()
+ }
+
+ fn build(self: Box<Self>) -> ArrayRef {
+ self.rows_to_array(&self.group_values)
+ }
+
+ fn take_n(&mut self, n: usize) -> ArrayRef {
+ debug_assert!(n <= self.group_values.num_rows());
+
+ // Materialize the first `n` group rows.
+ let output = self.rows_to_array(self.group_values.iter().take(n));
+
+ // Shift the remaining rows to the front by rebuilding the buffer.
+ // TODO: mirror the arrow-rs efficiency TODO in `GroupValuesRows::emit`.
+ let mut remaining = self.row_converter.empty_rows(0, 0);
+ for row in self.group_values.iter().skip(n) {
+ remaining.push(row);
+ }
+ self.group_values = remaining;
+
+ output
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use arrow::array::{Array, ArrayRef, FixedSizeListArray, Int32Array, StructArray};
+ use arrow::datatypes::{DataType, Field, Int32Type};
+ use std::sync::Arc;
+
+ fn fsl_i32(data: Vec<Option<Vec<Option<i32>>>>, list_len: i32) -> ArrayRef {
+ Arc::new(FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(
+ data, list_len,
+ ))
+ }
+
+ /// The generic column must agree with a per-row reference for equality,
+ /// including inner-null and outer-null rows, on a `FixedSizeList<Int32>`.
+ #[test]
+ fn fsl_append_equal_to_build_roundtrip() {
+ let dt = DataType::FixedSizeList(
+ Arc::new(Field::new("item", DataType::Int32, true)),
+ 2,
+ );
+ let mut col = Box::new(RowsGroupColumn::try_new(dt).unwrap());
+
+ // group values: [1,2], null-outer, [3, null-inner]
+ let input = fsl_i32(
+ vec![
+ Some(vec![Some(1), Some(2)]),
+ None,
+ Some(vec![Some(3), None]),
+ ],
+ 2,
+ );
+
+ col.vectorized_append(&input, &[0, 1, 2]).unwrap();
+ assert_eq!(col.len(), 3);
+
+ // Probe with a fresh batch: row0 == group0, row1 (null) == group1,
+ // row2 differs from group0, row3 (inner null) == group2.
+ let probe = fsl_i32(
+ vec![
+ Some(vec![Some(1), Some(2)]), // == g0
+ None, // == g1
+ Some(vec![Some(9), Some(9)]), // != g0
+ Some(vec![Some(3), None]), // == g2
+ ],
+ 2,
+ );
+
+ assert!(col.equal_to(0, &probe, 0));
+ assert!(col.equal_to(1, &probe, 1));
+ assert!(!col.equal_to(0, &probe, 2));
+ assert!(col.equal_to(2, &probe, 3));
+
+ // Vectorized equal_to should match the scalar reference.
+ let mut results = BooleanBufferBuilder::new(3);
+ results.append_n(3, true);
+ col.vectorized_equal_to(&[0, 1, 2], &probe, &[0, 1, 3], &mut results);
+ assert!(results.get_bit(0));
+ assert!(results.get_bit(1));
+ assert!(results.get_bit(2));
+
+ // build() must reproduce the original group values.
+ let out = col.build();
+ let out = out.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
+ assert_eq!(out.len(), 3);
+ assert!(out.is_null(1));
+ assert!(!out.is_null(0));
+ }
+
+ /// `take_n` must emit the first `n` rows and shift the rest to the front.
+ #[test]
+ fn fsl_take_n_shifts_remaining() {
+ let dt = DataType::FixedSizeList(
+ Arc::new(Field::new("item", DataType::Int32, true)),
+ 1,
+ );
+ let mut col = RowsGroupColumn::try_new(dt).unwrap();
+
+ let input = fsl_i32(
+ vec![
+ Some(vec![Some(10)]),
+ Some(vec![Some(20)]),
+ Some(vec![Some(30)]),
+ ],
+ 1,
+ );
+ col.vectorized_append(&input, &[0, 1, 2]).unwrap();
+
+ let first = col.take_n(1);
+ let first = first.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
+ let first_vals = first
+ .value(0)
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .unwrap()
+ .clone();
+ assert_eq!(first_vals.value(0), 10);
+ assert_eq!(col.len(), 2);
+
+ // Remaining 20, 30 should now be at indices 0, 1.
+ let rest = Box::new(col).build();
+ let rest = rest.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
+ assert_eq!(rest.len(), 2);
+ let g0 = rest
+ .value(0)
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .unwrap()
+ .value(0);
+ assert_eq!(g0, 20);
+ }
+
+ /// Works for `Struct<a: Int32>` too — proves the column is type-generic.
+ #[test]
+ fn struct_roundtrip() {
+ let dt = DataType::Struct(vec![Field::new("a", DataType::Int32, true)].into());
+ let mut col = RowsGroupColumn::try_new(dt).unwrap();
+
+ let a: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(2)]));
+ let input: ArrayRef = Arc::new(StructArray::new(
+ vec![Field::new("a", DataType::Int32, true)].into(),
+ vec![a],
+ None,
+ ));
+ col.vectorized_append(&input, &[0, 1]).unwrap();
+ assert_eq!(col.len(), 2);
+ assert!(col.equal_to(0, &input, 0));
+ assert!(!col.equal_to(0, &input, 1));
+ }
+
+ #[test]
+ fn supports_type_matches_row_converter_impl() {
+ assert!(RowsGroupColumn::supports_type(&DataType::FixedSizeList(
+ Arc::new(Field::new("item", DataType::Int32, true)),
+ 3
+ )));
+ assert!(RowsGroupColumn::supports_type(&DataType::Struct(
+ vec![Field::new("a", DataType::Int32, true)].into()
+ )));
+ // Whether Map is encodable depends on the arrow-rs version.
+ // Just assert that our `supports_type` agrees with arrow's
+ // `RowConverter::supports_fields` — either both accept it or both
+ // reject it. Both are correct wrt the invariant.
+ let map_field = Arc::new(Field::new(
+ "entries",
+ DataType::Struct(
+ vec![
+ Field::new("keys", DataType::Int32, false),
+ Field::new("values", DataType::Int32, true),
+ ]
+ .into(),
+ ),
+ false,
+ ));
+ let map_dt = DataType::Map(map_field, false);
+ let arrow_supports =
+ RowConverter::supports_fields(&[SortField::new(map_dt.clone())]);
+ assert_eq!(RowsGroupColumn::supports_type(&map_dt), arrow_supports);
+ }
+
+ /// Regression test for the nested-container recursion in
+ /// [`crate::aggregates::group_values::row::encode_array_if_necessary`].
+ /// `RowConverter` flattens dictionary values on the way in, so a
+ /// `List<Dict<Int32, Utf8>>` schema round-trips with `Utf8` values
+ /// unless the helper re-encodes the leaf. Without that recursion,
+ /// `build()` would emit an array whose data type does not match the
+ /// group column's declared type.
+ #[test]
+ fn build_preserves_list_of_dictionary_schema() {
+ use arrow::array::{DictionaryArray, ListArray, StringArray};
+ use arrow::buffer::OffsetBuffer;
+ use arrow::datatypes::Int32Type;
+
+ let dict_dt =
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+ let item_field = Arc::new(Field::new("item", dict_dt.clone(), true));
+ let outer_dt = DataType::List(Arc::clone(&item_field));
+
+ // Skip if this arrow-rs version rejects the nesting — the invariant we
+ // care about is `output().data_type() == declared type` conditional on
+ // supports_type saying yes.
+ if !RowsGroupColumn::supports_type(&outer_dt) {
+ return;
+ }
+
+ let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
+
+ // Build List<Dict<Int32,Utf8>> of one row = ["a", "b"].
+ let values = Arc::new(StringArray::from(vec!["a", "b"]));
+ let keys = Int32Array::from(vec![0, 1]);
+ let dict = DictionaryArray::<Int32Type>::try_new(keys, values).unwrap();
+ let offsets = OffsetBuffer::from_lengths([2]);
+ let list =
+ ListArray::try_new(Arc::clone(&item_field), offsets, Arc::new(dict), None)
+ .unwrap();
+ let input: ArrayRef = Arc::new(list);
+
+ col.vectorized_append(&input, &[0]).unwrap();
+ let built = col.build();
+ assert_eq!(
+ built.data_type(),
+ &outer_dt,
+ "build() must return the declared List<Dict> data type, \
+ not the RowConverter-flattened List<Utf8>",
+ );
+ }
+
+ // ---- FSL<Dict> rejection ----------------------------------------
+ //
+ // arrow-row 59.1.0's `decode_fixed_size_list` skips the
+ // dict-flatten correction that the generic `decode<L>` path applies
+ // to `List` / `LargeList` / `ListView` / `LargeListView` / `Map`,
+ // so any `FixedSizeList` containing a `Dictionary` leaf panics on
+ // emit. `supports_type` must reject those shapes so
+ // `GroupValuesRows` fallback handles them instead. These tests pin
+ // the current shape of that black-list.
+
+ fn dict_utf8() -> DataType {
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
+ }
+
+ fn fsl_of(inner: DataType) -> DataType {
+ DataType::FixedSizeList(Arc::new(Field::new("item", inner, true)), 2)
+ }
+
+ #[test]
+ fn supports_type_rejects_fixed_size_list_of_dict() {
+ // Direct case: `FixedSizeList<Dict<Int32, Utf8>>`.
+ assert!(!RowsGroupColumn::supports_type(&fsl_of(dict_utf8())));
+ }
+
+ #[test]
+ fn supports_type_rejects_fsl_with_dict_nested_in_struct() {
+ // The dict is one level deep under a struct that is itself the
+ // FSL element. arrow-row still panics because `convert_raw`
+ // returns the struct with a decoded (Utf8) field while the
+ // FSL builder expects the declared struct-with-dict shape.
+ let struct_dt = DataType::Struct(vec![Field::new("d", dict_utf8(), true)].into());
+ assert!(!RowsGroupColumn::supports_type(&fsl_of(struct_dt)));
+ }
+
+ #[test]
+ fn supports_type_rejects_fsl_with_dict_nested_in_list() {
+ // `FixedSizeList<List<Dict>>` — the inner `List<Dict>` handles
+ // dicts correctly on its own, but the outer FSL wrapper still
+ // panics with the mismatched declared child type.
+ let list_of_dict =
+ DataType::List(Arc::new(Field::new("item", dict_utf8(), true)));
+ assert!(!RowsGroupColumn::supports_type(&fsl_of(list_of_dict)));
+ }
+
+ #[test]
+ fn supports_type_rejects_fsl_hidden_under_outer_list() {
+ // Sibling positioning: the outer container is a `List` (which is
+ // fine on its own), but its child is a `FixedSizeList<Dict>`.
+ // The panic surface is at the inner FSL layer regardless of what
+ // wraps it, so this must still be rejected.
+ let outer =
+ DataType::List(Arc::new(Field::new("item", fsl_of(dict_utf8()), true)));
+ assert!(!RowsGroupColumn::supports_type(&outer));
+ }
+
+ #[test]
+ fn supports_type_rejects_fsl_hidden_under_outer_struct() {
+ // Same, but the outer wrapper is a struct.
+ let outer =
+ DataType::Struct(vec![Field::new("f", fsl_of(dict_utf8()), true)].into());
+ assert!(!RowsGroupColumn::supports_type(&outer));
+ }
+
+ // ---- FSL without dicts is still fine ----------------------------
+
+ #[test]
+ fn supports_type_accepts_fsl_of_primitive() {
+ // Sanity: a plain FSL<Int32> must not get caught by the
+ // dict-under-FSL blacklist.
+ assert!(RowsGroupColumn::supports_type(&fsl_of(DataType::Int32)));
+ }
+
+ #[test]
+ fn supports_type_accepts_fsl_of_struct_without_dict() {
+ // FSL of struct where the struct's fields are all primitives.
+ let struct_dt =
+ DataType::Struct(vec![Field::new("a", DataType::Int32, true)].into());
+ assert!(RowsGroupColumn::supports_type(&fsl_of(struct_dt)));
+ }
+
+ // ---- Positive round-trip tests for non-FSL list-likes -----------
+ //
+ // The other list-like decoders in arrow-row 59.1.0
+ // (`GenericListArrayOrMap` path) apply the corrected_type fix, so
+ // `List<Dict>`, `LargeList<Dict>`, `ListView<Dict>`, `LargeListView<Dict>`
+ // and `Map<..., Dict>` all round-trip cleanly. These tests pin
+ // that they are (a) accepted by `supports_type` and (b) actually
+ // survive `vectorized_append` + `build()` without panicking, so a
+ // future arrow-rs regression there is caught here rather than in
+ // production.
+
+ #[test]
+ fn supports_type_accepts_large_list_of_dict() {
+ let dt = DataType::LargeList(Arc::new(Field::new("item", dict_utf8(), true)));
+ assert!(RowsGroupColumn::supports_type(&dt));
+ }
+
+ #[test]
+ fn supports_type_accepts_list_view_of_dict() {
+ let dt = DataType::ListView(Arc::new(Field::new("item", dict_utf8(), true)));
+ assert!(RowsGroupColumn::supports_type(&dt));
+ }
+
+ #[test]
+ fn supports_type_accepts_large_list_view_of_dict() {
+ let dt = DataType::LargeListView(Arc::new(Field::new("item", dict_utf8(), true)));
+ assert!(RowsGroupColumn::supports_type(&dt));
+ }
+
+ #[test]
+ fn supports_type_map_agrees_with_row_converter() {
+ // Map<Int32, Dict<Int32, Utf8>>. Whether arrow-row supports Map
+ // depends on the version; either way, our `supports_type` must
+ // agree with `RowConverter::supports_fields` — otherwise we'd
+ // pick a strategy the converter can't back.
+ let entries = Arc::new(Field::new(
+ "entries",
+ DataType::Struct(
+ vec![
+ Field::new("keys", DataType::Int32, false),
+ Field::new("values", dict_utf8(), true),
+ ]
+ .into(),
+ ),
+ false,
+ ));
+ let map_dt = DataType::Map(entries, false);
+ let arrow_supports =
+ RowConverter::supports_fields(&[SortField::new(map_dt.clone())]);
+ assert_eq!(RowsGroupColumn::supports_type(&map_dt), arrow_supports);
+ }
+
+ /// End-to-end regression: `LargeList<Dict<Int32, Utf8>>` must
+ /// actually survive `vectorized_append` + `build()` on the current
+ /// arrow-rs version, not just be accepted by `supports_type`.
+ #[test]
+ fn build_preserves_large_list_of_dictionary_schema() {
+ use arrow::array::{DictionaryArray, LargeListArray, StringArray};
+ use arrow::buffer::OffsetBuffer;
+
+ let item_field = Arc::new(Field::new("item", dict_utf8(), true));
+ let outer_dt = DataType::LargeList(Arc::clone(&item_field));
+
+ // Skip if this arrow-rs version rejects the nesting (defensive:
+ // the invariant we care about is `output().data_type() == declared`
+ // conditional on `supports_type` saying yes).
+ if !RowsGroupColumn::supports_type(&outer_dt) {
+ return;
+ }
+
+ let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
+
+ let values = Arc::new(StringArray::from(vec!["a", "b"]));
+ let keys = Int32Array::from(vec![0, 1]);
+ let dict = DictionaryArray::<Int32Type>::try_new(keys, values).unwrap();
+ let offsets = OffsetBuffer::<i64>::from_lengths([2]);
+ let list = LargeListArray::try_new(
+ Arc::clone(&item_field),
+ offsets,
+ Arc::new(dict),
+ None,
+ )
+ .unwrap();
+
+ col.vectorized_append(&(Arc::new(list) as ArrayRef), &[0])
+ .unwrap();
+ let built = col.build();
+ assert_eq!(
+ built.data_type(),
+ &outer_dt,
+ "LargeList<Dict>: build() must preserve the declared type",
+ );
+ }
+
+ /// Build a two-row `ListView<Dict<Int32, Utf8>>` array with rows
+ /// `["a", "b"]` and `["c"]` — the shape from the review reproducer:
+ /// `arrow_cast(a, 'ListView(Dictionary(Int32, Utf8))')`.
+ fn list_view_of_dict_input() -> (DataType, ArrayRef) {
+ use arrow::array::{DictionaryArray, ListViewArray, StringArray};
+ use arrow::buffer::ScalarBuffer;
+
+ let item_field = Arc::new(Field::new("item", dict_utf8(), true));
+ let outer_dt = DataType::ListView(Arc::clone(&item_field));
+
+ let values = Arc::new(StringArray::from(vec!["a", "b", "c"]));
+ let keys = Int32Array::from(vec![0, 1, 2]);
+ let dict = DictionaryArray::<Int32Type>::try_new(keys, values).unwrap();
+ let offsets = ScalarBuffer::<i32>::from(vec![0, 2]);
+ let sizes = ScalarBuffer::<i32>::from(vec![2, 1]);
+ let list = ListViewArray::try_new(
+ Arc::clone(&item_field),
+ offsets,
+ sizes,
+ Arc::new(dict),
+ None,
+ )
+ .unwrap();
+ (outer_dt, Arc::new(list) as ArrayRef)
+ }
+
+ /// `ListView<Dict>`: arrow-row's `decode_list_view` flattens the
+ /// dictionary child (`corrected_type`), so `build` must re-encode
+ /// the emitted array back to the declared type. Regression for the
+ /// review reproducer that failed with
+ /// `expected ListView(Dictionary(Int32, Utf8)) but found ListView(Utf8)`.
+ #[test]
+ fn build_preserves_list_view_of_dictionary_schema() {
+ let (outer_dt, input) = list_view_of_dict_input();
+ if !RowsGroupColumn::supports_type(&outer_dt) {
+ return;
+ }
+
+ let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
+ col.vectorized_append(&input, &[0, 1]).unwrap();
+ assert_eq!(col.len(), 2);
+
+ let built = col.build();
+ assert_eq!(
+ built.data_type(),
+ &outer_dt,
+ "ListView<Dict>: build() must return the declared type, \
+ not the RowConverter-flattened ListView<Utf8>",
+ );
+ assert_eq!(built.len(), 2);
+ }
+
+ /// Same regression through the `take_n` path (used by
+ /// `EmitTo::First(n)`), including the type of the *remaining*
+ /// values emitted by a subsequent `build`.
+ #[test]
+ fn take_n_preserves_list_view_of_dictionary_schema() {
+ let (outer_dt, input) = list_view_of_dict_input();
+ if !RowsGroupColumn::supports_type(&outer_dt) {
+ return;
+ }
+
+ let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
+ col.vectorized_append(&input, &[0, 1]).unwrap();
+
+ let taken = col.take_n(1);
+ assert_eq!(
+ taken.data_type(),
+ &outer_dt,
+ "ListView<Dict>: take_n() must return the declared type",
+ );
+ assert_eq!(taken.len(), 1);
+
+ let rest = col.build();
+ assert_eq!(
+ rest.data_type(),
+ &outer_dt,
+ "ListView<Dict>: build() after take_n must also preserve the type",
+ );
+ assert_eq!(rest.len(), 1);
+ }
+
+ /// `LargeListView<Dict>` fails the same way as `ListView<Dict>`
+ /// per the review; cover both `build` and `take_n`.
+ #[test]
+ fn build_and_take_n_preserve_large_list_view_of_dictionary_schema() {
+ use arrow::array::{DictionaryArray, LargeListViewArray, StringArray};
+ use arrow::buffer::ScalarBuffer;
+
+ let item_field = Arc::new(Field::new("item", dict_utf8(), true));
+ let outer_dt = DataType::LargeListView(Arc::clone(&item_field));
+ if !RowsGroupColumn::supports_type(&outer_dt) {
+ return;
+ }
+
+ let values = Arc::new(StringArray::from(vec!["a", "b", "c"]));
+ let keys = Int32Array::from(vec![0, 1, 2]);
+ let dict = DictionaryArray::<Int32Type>::try_new(keys, values).unwrap();
+ let offsets = ScalarBuffer::<i64>::from(vec![0, 2]);
+ let sizes = ScalarBuffer::<i64>::from(vec![2, 1]);
+ let list = LargeListViewArray::try_new(
+ Arc::clone(&item_field),
+ offsets,
+ sizes,
+ Arc::new(dict),
+ None,
+ )
+ .unwrap();
+ let input: ArrayRef = Arc::new(list);
+
+ let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
+ col.vectorized_append(&input, &[0, 1]).unwrap();
+
+ let taken = col.take_n(1);
+ assert_eq!(
+ taken.data_type(),
+ &outer_dt,
+ "LargeListView<Dict>: take_n() must return the declared type",
+ );
+
+ let rest = col.build();
+ assert_eq!(
+ rest.data_type(),
+ &outer_dt,
+ "LargeListView<Dict>: build() must return the declared type",
+ );
+ assert_eq!(rest.len(), 1);
+ }
+
+ /// Group-identity must survive the dictionary flatten + re-encode
+ /// round trip: appending the same logical list twice (with distinct
+ /// dictionary key mappings) must map to one group, a different list
+ /// to another. Mirrors the review reproducer's GROUP BY semantics
+ /// (2 distinct groups from 3 input rows).
+ #[test]
+ fn list_view_of_dict_groups_by_logical_value() {
+ use arrow::array::{DictionaryArray, ListViewArray, StringArray};
+ use arrow::buffer::ScalarBuffer;
+
+ let item_field = Arc::new(Field::new("item", dict_utf8(), true));
+ let outer_dt = DataType::ListView(Arc::clone(&item_field));
+ if !RowsGroupColumn::supports_type(&outer_dt) {
+ return;
+ }
+
+ // Rows: ["a","b"], ["a","b"], ["c"] → 2 distinct groups.
+ let values = Arc::new(StringArray::from(vec!["a", "b", "a", "b", "c"]));
+ let keys = Int32Array::from(vec![0, 1, 2, 3, 4]);
+ let dict = DictionaryArray::<Int32Type>::try_new(keys, values).unwrap();
+ let offsets = ScalarBuffer::<i32>::from(vec![0, 2, 4]);
+ let sizes = ScalarBuffer::<i32>::from(vec![2, 2, 1]);
+ let list = ListViewArray::try_new(
+ Arc::clone(&item_field),
+ offsets,
+ sizes,
+ Arc::new(dict),
+ None,
+ )
+ .unwrap();
+ let input: ArrayRef = Arc::new(list);
+
+ let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
+ // Append row 0 as group 0.
+ col.vectorized_append(&input, &[0]).unwrap();
+ // Row 1 must compare equal to group 0 (same logical value).
+ assert!(
+ col.equal_to(0, &input, 1),
+ "identical logical lists must be equal regardless of dict keys",
+ );
+ // Row 2 must not.
+ assert!(
+ !col.equal_to(0, &input, 2),
+ "different logical lists must not be equal",
+ );
+
+ col.vectorized_append(&input, &[2]).unwrap();
+ assert_eq!(col.len(), 2, "3 input rows → 2 distinct groups");
+
+ let built = col.build();
+ assert_eq!(built.data_type(), &outer_dt);
+ assert_eq!(built.len(), 2);
+ }
+
+ /// End-to-end regression for `Map<Int32, Dict<Int32, Utf8>>` when
+ /// arrow-row supports it. Same intent as the LargeList test.
+ #[test]
+ fn build_preserves_map_of_dictionary_schema() {
+ use arrow::array::{
+ DictionaryArray, Int32Array, MapArray, StringArray, StructArray,
+ };
+ use arrow::buffer::OffsetBuffer;
+
+ let key_field = Arc::new(Field::new("keys", DataType::Int32, false));
+ let value_field = Arc::new(Field::new("values", dict_utf8(), true));
+ let entries_field = Arc::new(Field::new(
+ "entries",
+ DataType::Struct(vec![(*key_field).clone(), (*value_field).clone()].into()),
+ false,
+ ));
+ let outer_dt = DataType::Map(Arc::clone(&entries_field), false);
+
+ if !RowsGroupColumn::supports_type(&outer_dt) {
+ return;
+ }
+
+ let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap());
+
+ // One map entry: {1 -> "a"}.
+ let keys = Arc::new(Int32Array::from(vec![1])) as ArrayRef;
+ let values_arr = Arc::new(StringArray::from(vec!["a"]));
+ let value_keys = Int32Array::from(vec![0]);
+ let value_dict =
+ DictionaryArray::<Int32Type>::try_new(value_keys, values_arr).unwrap();
+ let entries = StructArray::try_new(
+ vec![(*key_field).clone(), (*value_field).clone()].into(),
+ vec![keys, Arc::new(value_dict)],
+ None,
+ )
+ .unwrap();
+ let offsets = OffsetBuffer::<i32>::from_lengths([1]);
+ let map =
+ MapArray::try_new(Arc::clone(&entries_field), offsets, entries, None, false)
+ .unwrap();
+
+ col.vectorized_append(&(Arc::new(map) as ArrayRef), &[0])
+ .unwrap();
+ let built = col.build();
+ assert_eq!(
+ built.data_type(),
+ &outer_dt,
+ "Map<..., Dict>: build() must preserve the declared type",
+ );
+ }
+
+ // ---- Union / RunEndEncoded defensive rejection -----------------
+ //
+ // Before this PR both types were routed to `GroupValuesRows`
+ // (`group_column_supported_type` didn't have a nested branch). This
+ // PR added `is_nested`-based dispatch to `RowsGroupColumn`, which
+ // would opt them in — but the arrow-row round-trip for these two
+ // families hasn't been covered by our tests. Reject them here so
+ // the pre-PR routing is preserved; drop the blacklist when the
+ // round-trip matrix grows to include them.
+
+ #[test]
+ fn supports_type_rejects_union() {
+ use arrow::datatypes::UnionFields;
+
+ let fields = UnionFields::try_new(
+ vec![0_i8, 1_i8],
+ vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Utf8, true),
+ ],
+ )
+ .unwrap();
+ let dt = DataType::Union(fields, arrow::datatypes::UnionMode::Dense);
+ assert!(
+ !RowsGroupColumn::supports_type(&dt),
+ "Union must fall back to GroupValuesRows until arrow-row \
+ round-trip is covered by our tests",
+ );
+ }
+
+ #[test]
+ fn supports_type_rejects_run_end_encoded_with_nested_values() {
+ // REE with `is_nested() = true` (nested values) is what this PR
+ // could otherwise opt into RowsGroupColumn; keep it on
+ // GroupValuesRows.
+ let list_of_i32 =
+ DataType::List(Arc::new(Field::new("item", DataType::Int32, true)));
+ let dt = DataType::RunEndEncoded(
+ Arc::new(Field::new("run_ends", DataType::Int32, false)),
+ Arc::new(Field::new("values", list_of_i32, true)),
+ );
+ assert!(!RowsGroupColumn::supports_type(&dt));
+ }
+
+ #[test]
+ fn supports_type_rejects_run_end_encoded_with_scalar_values() {
+ // REE with scalar values is `is_nested() == false`, so
+ // `group_column_supported_type` never routes it to us via the
+ // nested branch anyway — but pin the invariant explicitly so a
+ // future refactor doesn't accidentally opt it in.
+ let dt = DataType::RunEndEncoded(
+ Arc::new(Field::new("run_ends", DataType::Int32, false)),
+ Arc::new(Field::new("values", DataType::Utf8, true)),
+ );
+ assert!(!RowsGroupColumn::supports_type(&dt));
+ }
+
+ #[test]
+ fn supports_type_rejects_ree_hidden_under_outer_wrapper() {
+ // REE buried under a struct or list: still rejected because
+ // the wrapper's decoder recurses through the REE branch we
+ // haven't validated.
+ let ree = DataType::RunEndEncoded(
+ Arc::new(Field::new("run_ends", DataType::Int32, false)),
+ Arc::new(Field::new("values", DataType::Utf8, true)),
+ );
+ let outer = DataType::Struct(vec![Field::new("f", ree, true)].into());
+ assert!(!RowsGroupColumn::supports_type(&outer));
+ }
+
+ #[test]
+ fn supports_type_accepts_plain_list_and_struct_still() {
+ // Sanity: the defensive Union/REE blacklist must not accidentally
+ // catch the well-tested list-likes / structs that this column
+ // exists to serve.
+ let list_of_int =
+ DataType::List(Arc::new(Field::new("item", DataType::Int32, true)));
+ assert!(RowsGroupColumn::supports_type(&list_of_int));
+
+ let struct_of_prims = DataType::Struct(
+ vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Utf8, true),
+ ]
+ .into(),
+ );
+ assert!(RowsGroupColumn::supports_type(&struct_of_prims));
+ }
+}
diff --git a/datafusion/physical-plan/src/aggregates/group_values/row.rs b/datafusion/physical-plan/src/aggregates/group_values/row.rs
index 4976a098ec..cbd7a609c5 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/row.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/row.rs
@@ -17,7 +17,8 @@
use crate::aggregates::group_values::GroupValues;
use arrow::array::{
- Array, ArrayRef, ListArray, PrimitiveArray, RunArray, StructArray,
+ Array, ArrayRef, FixedSizeListArray, LargeListArray, LargeListViewArray, ListArray,
+ ListViewArray, MapArray, PrimitiveArray, RunArray, StructArray,
downcast_run_end_index,
};
use arrow::compute::cast;
@@ -247,7 +248,7 @@ impl GroupValues for GroupValuesRows {
// https://github.com/apache/datafusion/issues/7647
for (field, array) in self.schema.fields.iter().zip(&mut output) {
let expected = field.data_type();
- *array = dictionary_encode_if_necessary(array, expected)?;
+ *array = encode_array_if_necessary(array, expected)?;
}
self.group_values = Some(group_values);
@@ -267,7 +268,17 @@ impl GroupValues for GroupValuesRows {
}
}
-fn dictionary_encode_if_necessary(
+/// Re-apply dictionary / run-end encoding to `array` so it matches `expected`.
+///
+/// Arrow's [`RowConverter`] flattens dictionary and run-end-encoded values to
+/// their plain value type during row encoding (at [`RowConverter::append`]),
+/// so any group-value array produced from the row format is in that plain
+/// type and must be re-encoded to match the schema's expected type before
+/// being returned. Shared with the generic row-backed `GroupColumn`.
+///
+/// [`RowConverter`]: arrow::row::RowConverter
+/// [`RowConverter::append`]: arrow::row::RowConverter::append
+pub(crate) fn encode_array_if_necessary(
array: &ArrayRef,
expected: &DataType,
) -> Result<ArrayRef> {
@@ -278,7 +289,7 @@ fn dictionary_encode_if_necessary(
.iter()
.zip(struct_array.columns())
.map(|(expected_field, column)| {
- dictionary_encode_if_necessary(column, expected_field.data_type())
+ encode_array_if_necessary(column, expected_field.data_type())
})
.collect::<Result<Vec<_>>>()?;
@@ -294,13 +305,82 @@ fn dictionary_encode_if_necessary(
Ok(Arc::new(ListArray::try_new(
Arc::<arrow::datatypes::Field>::clone(expected_field),
list.offsets().clone(),
- dictionary_encode_if_necessary(
- list.values(),
- expected_field.data_type(),
- )?,
+ encode_array_if_necessary(list.values(), expected_field.data_type())?,
+ list.nulls().cloned(),
+ )?))
+ }
+ (DataType::LargeList(expected_field), &DataType::LargeList(_)) => {
+ let list = array.as_any().downcast_ref::<LargeListArray>().unwrap();
+
+ Ok(Arc::new(LargeListArray::try_new(
+ Arc::<arrow::datatypes::Field>::clone(expected_field),
+ list.offsets().clone(),
+ encode_array_if_necessary(list.values(), expected_field.data_type())?,
+ list.nulls().cloned(),
+ )?))
+ }
+ (DataType::ListView(expected_field), &DataType::ListView(_)) => {
+ // arrow-row's `decode_list_view` applies the dictionary-flatten
+ // `corrected_type` to the child, so a `ListView<Dictionary<..>>`
+ // decodes as `ListView<value type>` and the child must be
+ // re-encoded here (same as `List` above, plus the `sizes`
+ // buffer that view-lists carry).
+ let list = array.as_any().downcast_ref::<ListViewArray>().unwrap();
+
+ Ok(Arc::new(ListViewArray::try_new(
+ Arc::<arrow::datatypes::Field>::clone(expected_field),
+ list.offsets().clone(),
+ list.sizes().clone(),
+ encode_array_if_necessary(list.values(), expected_field.data_type())?,
+ list.nulls().cloned(),
+ )?))
+ }
+ (DataType::LargeListView(expected_field), &DataType::LargeListView(_)) => {
+ let list = array.as_any().downcast_ref::<LargeListViewArray>().unwrap();
+
+ Ok(Arc::new(LargeListViewArray::try_new(
+ Arc::<arrow::datatypes::Field>::clone(expected_field),
+ list.offsets().clone(),
+ list.sizes().clone(),
+ encode_array_if_necessary(list.values(), expected_field.data_type())?,
+ list.nulls().cloned(),
+ )?))
+ }
+ (
+ DataType::FixedSizeList(expected_field, expected_size),
+ &DataType::FixedSizeList(_, _),
+ ) => {
+ let list = array.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
+
+ Ok(Arc::new(FixedSizeListArray::try_new(
+ Arc::<arrow::datatypes::Field>::clone(expected_field),
+ *expected_size,
+ encode_array_if_necessary(list.values(), expected_field.data_type())?,
list.nulls().cloned(),
)?))
}
+ (DataType::Map(expected_entries_field, ordered), &DataType::Map(_, _)) => {
+ let map = array.as_any().downcast_ref::<MapArray>().unwrap();
+ // Re-encode the entries `StructArray` (which holds key/value
+ // columns) against the expected entries field's struct type.
+ let entries_as_ref: ArrayRef = Arc::new(map.entries().clone());
+ let entries = encode_array_if_necessary(
+ &entries_as_ref,
+ expected_entries_field.data_type(),
+ )?;
+ let entries = entries
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .expect("Map entries recurse must yield a StructArray")
+ .clone();
+ Ok(Arc::new(MapArray::try_new(
+ Arc::<arrow::datatypes::Field>::clone(expected_entries_field),
+ map.offsets().clone(),
+ entries,
+ map.nulls().cloned(),
+ *ordered,
+ )?))
+ }
(DataType::Dictionary(_, _), _) => Ok(cast(array.as_ref(), expected)?),
(
DataType::RunEndEncoded(run_ends_field, expected_values_field),
@@ -312,7 +392,7 @@ fn dictionary_encode_if_necessary(
.as_any()
.downcast_ref::<RunArray<$run_end_type>>()
.unwrap();
- let values = dictionary_encode_if_necessary(
+ let values = encode_array_if_necessary(
&(Arc::clone(run_array.values()) as ArrayRef),
expected_values_field.data_type(),
)?;