Apache DataFusion SQL Query Engine
feat: add GroupColumn support for Decimal256 in multi-column GROUP BY (#23849)
Varun038bfa22a5cb
diff --git a/datafusion/physical-plan/benches/multi_group_by.rs b/datafusion/physical-plan/benches/multi_group_by.rsindex 11481d4f91..0c689f9fcb 100644--- a/datafusion/physical-plan/benches/multi_group_by.rs+++ b/datafusion/physical-plan/benches/multi_group_by.rs@@ -28,12 +28,13 @@//! `FixedSizeBinaryGroupValueBuilder`.use arrow::array::{- ArrayRef, DurationMicrosecondArray, Float16Array, Int32Array,+ ArrayRef, Decimal256Array, DurationMicrosecondArray, Float16Array, Int32Array,IntervalMonthDayNanoArray, UInt32Array,};use arrow::compute::take;use arrow::datatypes::{DataType, Field, IntervalMonthDayNano, IntervalUnit, Schema, SchemaRef, TimeUnit,+ i256,};use arrow::util::bench_util::create_fsb_array;use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};@@ -709,6 +710,94 @@ fn bench_interval(c: &mut Criterion) {group.finish();}+fn make_decimal256_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("dec", DataType::Decimal256(50, 0), false),+ Field::new("id", DataType::Int32, false),+ ]))+}++/// Generate `(Decimal256(50, 0), Int32)` batches with `num_distinct_groups`+/// distinct keys.+///+/// Each distinct value is `i256::from_i128(g)`, and precision > 38 keeps it a+/// genuine `Decimal256`. The `Int32` column is keyed identically so the combined+/// cardinality equals `num_distinct_groups`.+fn generate_decimal256_batches(+ num_distinct_groups: usize,+ num_rows: usize,+ batch_size: usize,+) -> Vec<Vec<ArrayRef>> {+ let num_full_batches = num_rows / batch_size;+ let remainder = num_rows % batch_size;+ let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };++ (0..num_batches)+ .map(|batch_idx| {+ let batch_start = batch_idx * batch_size;+ let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {+ remainder+ } else {+ batch_size+ };++ let group_ids = (0..current_batch_size)+ .map(|row| (batch_start + row) % num_distinct_groups);++ let keys = Decimal256Array::from_iter_values(+ group_ids.clone().map(|g| i256::from_i128(g as i128)),+ )+ .with_precision_and_scale(50, 0)+ .unwrap();+ let id: Int32Array = group_ids.map(|g| g as i32).collect();++ vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]+ })+ .collect()+}++/// Experiment 11: Group count sweep for a `(Decimal256, Int32)` key.+///+/// Exercises the primitive `GroupColumn` builder for `Decimal256` (32-byte+/// `i256` native) on the multi-column path (previously such a schema fell back+/// to `GroupValuesRows`).+fn bench_decimal256(c: &mut Criterion) {+ let mut group = c.benchmark_group("decimal256");+ group.sample_size(15);++ let schema = make_decimal256_schema();++ for num_groups in [1_000, 1_000_000] {+ let batches =+ generate_decimal256_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);++ for vectorized in [true, false] {+ let label = if vectorized {+ "vectorized"+ } else {+ "row_based"+ };+ group.bench_with_input(+ BenchmarkId::new(label, format!("grp_{num_groups}")),+ &batches,+ |b, batches| {+ b.iter_batched_ref(+ || {+ (+ create_group_values(&schema, vectorized),+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),+ )+ },+ |(gv, groups)| bench_intern(gv, batches, groups),+ criterion::BatchSize::LargeInput,+ );+ },+ );+ }+ }+ group.finish();+}+criterion_group!(benches,bench_issue_17850_regression,@@ -721,5 +810,6 @@ criterion_group!(bench_float16,bench_duration,bench_interval,+ bench_decimal256,);criterion_main!(benches);diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rsindex 8b68152c47..a3ac23c15e 100644--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs@@ -34,7 +34,7 @@ use crate::aggregates::group_values::multi_group_by::{use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};use arrow::compute::cast;use arrow::datatypes::{- BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type,+ BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType,DurationSecondType, Field, Float16Type, Float32Type, Float64Type, Int8Type,Int16Type, Int32Type, Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType,@@ -951,6 +951,7 @@ fn group_column_supported_type(data_type: &DataType) -> bool {| DataType::Float32| DataType::Float64| DataType::Decimal128(_, _)+ | DataType::Decimal256(_, _)| DataType::Utf8| DataType::LargeUtf8| DataType::Binary@@ -1081,6 +1082,9 @@ fn make_group_column(field: &Field) -> Result<Box<dyn GroupColumn>> {DataType::Decimal128(_, _) => {instantiate_primitive!(v, nullable, Decimal128Type, data_type)}+ DataType::Decimal256(_, _) => {+ instantiate_primitive!(v, nullable, Decimal256Type, data_type)+ }DataType::Utf8 => {v.push(Box::new(ByteGroupValueBuilder::<i32>::new(OutputType::Utf8,@@ -1603,6 +1607,7 @@ mod tests {DataType::Float64,DataType::Float16,DataType::Decimal128(38, 10),+ DataType::Decimal256(76, 10),DataType::Utf8,DataType::LargeUtf8,DataType::Utf8View,@@ -1640,7 +1645,6 @@ mod tests {}let unsupported_cases: Vec<DataType> = vec![- DataType::Decimal256(76, 10),// Invalid Time-unit combinations: Time32 is defined only for// Second / Millisecond and Time64 only for Microsecond /// Nanosecond. The TimeUnit enum allows constructing the otherdiff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.sltindex 637b7c1882..38d1b78214 100644--- a/datafusion/sqllogictest/test_files/group_by.slt+++ b/datafusion/sqllogictest/test_files/group_by.slt@@ -5765,3 +5765,19 @@ FROM interval_group_test GROUP BY column1, column2 ORDER BY column1, count(*);statement okDROP TABLE interval_group_test;++# Test multi group by int + Decimal256+statement ok+create table decimal256_multi_group (k int, d decimal(50, 2)) as values+ (1, 100.00), (1, 100.00), (1, 250.00), (2, 100.00), (2, NULL);++query IRI+select k, d, count(*) from decimal256_multi_group group by k, d order by k, d;+----+1 100 2+1 250 1+2 100 1+2 NULL 1++statement ok+drop table decimal256_multi_group;