Skip to content

Apache DataFusion SQL Query Engine

Fix panic for `GROUPING SETS(())` and handle empty-grouping aggregates (#19252)

kosiewc53a44891bb5
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index 5bce862dc9..3e216c619b 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -1582,7 +1582,8 @@ impl DefaultPhysicalPlanner {
}
} else if group_expr.is_empty() {
// No GROUP BY clause - create empty PhysicalGroupBy
- Ok(PhysicalGroupBy::new(vec![], vec![], vec![]))
+ // no expressions, no null expressions and no grouping expressions
+ Ok(PhysicalGroupBy::new(vec![], vec![], vec![], false))
} else {
Ok(PhysicalGroupBy::new_single(
group_expr
@@ -1654,6 +1655,7 @@ fn merge_grouping_set_physical_expr(
grouping_set_expr,
null_exprs,
merged_sets,
+ true,
))
}
@@ -1696,7 +1698,7 @@ fn create_cube_physical_expr(
}
}
- Ok(PhysicalGroupBy::new(all_exprs, null_exprs, groups))
+ Ok(PhysicalGroupBy::new(all_exprs, null_exprs, groups, true))
}
/// Expand and align a ROLLUP expression. This is a special case of GROUPING SETS
@@ -1741,7 +1743,7 @@ fn create_rollup_physical_expr(
groups.push(group)
}
- Ok(PhysicalGroupBy::new(all_exprs, null_exprs, groups))
+ Ok(PhysicalGroupBy::new(all_exprs, null_exprs, groups, true))
}
/// For a given logical expr, get a properly typed NULL ScalarValue physical expression
@@ -2832,6 +2834,7 @@ mod tests {
true,
],
],
+ has_grouping_set: true,
},
)
"#);
@@ -2942,6 +2945,7 @@ mod tests {
false,
],
],
+ has_grouping_set: true,
},
)
"#);
diff --git a/datafusion/core/tests/execution/coop.rs b/datafusion/core/tests/execution/coop.rs
index b6f406e967..fb9495da50 100644
--- a/datafusion/core/tests/execution/coop.rs
+++ b/datafusion/core/tests/execution/coop.rs
@@ -170,7 +170,7 @@ async fn agg_no_grouping_yields(
let inf = Arc::new(make_lazy_exec("value", pretend_infinite));
let aggr = Arc::new(AggregateExec::try_new(
AggregateMode::Single,
- PhysicalGroupBy::new(vec![], vec![], vec![]),
+ PhysicalGroupBy::new(vec![], vec![], vec![], false),
vec![Arc::new(
AggregateExprBuilder::new(
sum::sum_udaf(),
@@ -204,7 +204,7 @@ async fn agg_grouping_yields(
let aggr = Arc::new(AggregateExec::try_new(
AggregateMode::Single,
- PhysicalGroupBy::new(vec![(group, "group".to_string())], vec![], vec![]),
+ PhysicalGroupBy::new(vec![(group, "group".to_string())], vec![], vec![], false),
vec![Arc::new(
AggregateExprBuilder::new(sum::sum_udaf(), vec![value_col.clone()])
.schema(inf.schema())
@@ -240,6 +240,7 @@ async fn agg_grouped_topk_yields(
vec![(group, "group".to_string())],
vec![],
vec![vec![false]],
+ false,
),
vec![Arc::new(
AggregateExprBuilder::new(min_max::max_udaf(), vec![value_col.clone()])
@@ -545,6 +546,7 @@ async fn interleave_then_aggregate_yields(
vec![], // no GROUP BY columns
vec![], // no GROUP BY expressions
vec![], // no GROUP BY physical expressions
+ false,
),
vec![Arc::new(aggregate_expr)],
vec![None], // no “distinct” flags
@@ -676,7 +678,7 @@ async fn join_agg_yields(
let aggr = Arc::new(AggregateExec::try_new(
AggregateMode::Single,
- PhysicalGroupBy::new(vec![], vec![], vec![]),
+ PhysicalGroupBy::new(vec![], vec![], vec![], false),
vec![Arc::new(aggregate_expr)],
vec![None],
projection,
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs
index 02ff825065..0956974cfc 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs
@@ -2545,6 +2545,7 @@ fn test_no_pushdown_grouping_sets_filter_on_missing_column() {
vec![false, false], // (a, b) - both present
vec![true, false], // (b) - a is NULL, b present
],
+ true,
);
let aggregate = Arc::new(
@@ -2615,6 +2616,7 @@ fn test_pushdown_grouping_sets_filter_on_common_column() {
vec![false, false], // (a, b) - both present
vec![true, false], // (b) - a is NULL, b present
],
+ true,
);
let aggregate = Arc::new(
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 3dbcce699c..447302d762 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -177,6 +177,9 @@ pub struct PhysicalGroupBy {
/// expression in null_expr. If `groups[i][j]` is true, then the
/// j-th expression in the i-th group is NULL, otherwise it is `expr[j]`.
groups: Vec<Vec<bool>>,
+ /// True when GROUPING SETS/CUBE/ROLLUP are used so `__grouping_id` should
+ /// be included in the output schema.
+ has_grouping_set: bool,
}
impl PhysicalGroupBy {
@@ -185,11 +188,13 @@ impl PhysicalGroupBy {
expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
null_expr: Vec<(Arc<dyn PhysicalExpr>, String)>,
groups: Vec<Vec<bool>>,
+ has_grouping_set: bool,
) -> Self {
Self {
expr,
null_expr,
groups,
+ has_grouping_set,
}
}
@@ -201,6 +206,7 @@ impl PhysicalGroupBy {
expr,
null_expr: vec![],
groups: vec![vec![false; num_exprs]],
+ has_grouping_set: false,
}
}
@@ -217,6 +223,11 @@ impl PhysicalGroupBy {
exprs_nullable
}
+ /// Returns true if this has no grouping at all (including no GROUPING SETS)
+ pub fn is_true_no_grouping(&self) -> bool {
+ self.is_empty() && !self.has_grouping_set
+ }
+
/// Returns the group expressions
pub fn expr(&self) -> &[(Arc<dyn PhysicalExpr>, String)] {
&self.expr
@@ -232,14 +243,20 @@ impl PhysicalGroupBy {
&self.groups
}
+ /// Returns true if this grouping uses GROUPING SETS, CUBE or ROLLUP.
+ pub fn has_grouping_set(&self) -> bool {
+ self.has_grouping_set
+ }
+
/// Returns true if this `PhysicalGroupBy` has no group expressions
pub fn is_empty(&self) -> bool {
self.expr.is_empty()
}
- /// Check whether grouping set is single group
+ /// Returns true if this is a "simple" GROUP BY (not using GROUPING SETS/CUBE/ROLLUP).
+ /// This determines whether the `__grouping_id` column is included in the output schema.
pub fn is_single(&self) -> bool {
- self.null_expr.is_empty()
+ !self.has_grouping_set
}
/// Calculate GROUP BY expressions according to input schema.
@@ -253,7 +270,7 @@ impl PhysicalGroupBy {
/// The number of expressions in the output schema.
fn num_output_exprs(&self) -> usize {
let mut num_exprs = self.expr.len();
- if !self.is_single() {
+ if self.has_grouping_set {
num_exprs += 1
}
num_exprs
@@ -270,7 +287,7 @@ impl PhysicalGroupBy {
.take(num_output_exprs)
.map(|(index, (_, name))| Arc::new(Column::new(name, index)) as _),
);
- if !self.is_single() {
+ if self.has_grouping_set {
output_exprs.push(Arc::new(Column::new(
Aggregate::INTERNAL_GROUPING_ID,
self.expr.len(),
@@ -281,11 +298,7 @@ impl PhysicalGroupBy {
/// Returns the number expression as grouping keys.
pub fn num_group_exprs(&self) -> usize {
- if self.is_single() {
- self.expr.len()
- } else {
- self.expr.len() + 1
- }
+ self.expr.len() + usize::from(self.has_grouping_set)
}
pub fn group_schema(&self, schema: &Schema) -> Result<SchemaRef> {
@@ -308,7 +321,7 @@ impl PhysicalGroupBy {
.into(),
);
}
- if !self.is_single() {
+ if self.has_grouping_set {
fields.push(
Field::new(
Aggregate::INTERNAL_GROUPING_ID,
@@ -344,17 +357,17 @@ impl PhysicalGroupBy {
)
.collect();
let num_exprs = expr.len();
- let groups = if self.expr.is_empty() {
+ let groups = if self.expr.is_empty() && !self.has_grouping_set {
// No GROUP BY expressions - should have no groups
vec![]
} else {
- // Has GROUP BY expressions - create a single group
vec![vec![false; num_exprs]]
};
Self {
expr,
null_expr: vec![],
groups,
+ has_grouping_set: false,
}
}
}
@@ -374,6 +387,7 @@ impl PartialEq for PhysicalGroupBy {
.zip(other.null_expr.iter())
.all(|((expr1, name1), (expr2, name2))| expr1.eq(expr2) && name1 == name2)
&& self.groups == other.groups
+ && self.has_grouping_set == other.has_grouping_set
}
}
@@ -723,8 +737,7 @@ impl AggregateExec {
partition: usize,
context: &Arc<TaskContext>,
) -> Result<StreamType> {
- // no group by at all
- if self.group_by.expr.is_empty() {
+ if self.group_by.is_true_no_grouping() {
return Ok(StreamType::AggregateStream(AggregateStream::new(
self, context, partition,
)?));
@@ -757,7 +770,7 @@ impl AggregateExec {
/// on an AggregateExec.
pub fn is_unordered_unfiltered_group_by_distinct(&self) -> bool {
// ensure there is a group by
- if self.group_expr().is_empty() {
+ if self.group_expr().is_empty() && !self.group_expr().has_grouping_set() {
return false;
}
// ensure there are no aggregate expressions
@@ -1954,6 +1967,7 @@ mod tests {
vec![true, false], // (NULL, b)
vec![false, false], // (a,b)
],
+ true,
);
let aggregates = vec![Arc::new(
@@ -2103,6 +2117,7 @@ mod tests {
vec![(col("a", &input_schema)?, "a".to_string())],
vec![],
vec![vec![false]],
+ false,
);
let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(
@@ -2448,6 +2463,7 @@ mod tests {
vec![(col("a", &input_schema)?, "a".to_string())],
vec![],
vec![vec![false]],
+ false,
);
// something that allocates within the aggregator
@@ -2892,6 +2908,7 @@ mod tests {
vec![true, false, true],
vec![true, true, false],
],
+ true,
);
let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![
@@ -3251,6 +3268,7 @@ mod tests {
vec![false, true], // (a, NULL)
vec![false, false], // (a,b)
],
+ true,
);
let aggr_schema = create_schema(
&input_schema,
@@ -3302,6 +3320,7 @@ mod tests {
vec![(col("a", &schema)?, "a".to_string())],
vec![],
vec![vec![false]],
+ false,
);
// Test with MIN for simple intermediate state (min) and AVG for multiple intermediate states (partial sum, partial count).
diff --git a/datafusion/physical-plan/src/aggregates/no_grouping.rs b/datafusion/physical-plan/src/aggregates/no_grouping.rs
index 291d68d319..a55d70ca6f 100644
--- a/datafusion/physical-plan/src/aggregates/no_grouping.rs
+++ b/datafusion/physical-plan/src/aggregates/no_grouping.rs
@@ -251,6 +251,23 @@ fn scalar_cmp_null_short_circuit(
}
}
+/// Prepend the grouping ID column to the output columns if present.
+///
+/// For GROUPING SETS with no GROUP BY expressions, the schema includes a `__grouping_id`
+/// column that must be present in the output. This function inserts it at the beginning
+/// of the columns array to maintain schema alignment.
+fn prepend_grouping_id_column(
+ mut columns: Vec<Arc<dyn arrow::array::Array>>,
+ grouping_id: Option<&ScalarValue>,
+) -> Result<Vec<Arc<dyn arrow::array::Array>>> {
+ if let Some(id) = grouping_id {
+ let num_rows = columns.first().map(|array| array.len()).unwrap_or(1);
+ let grouping_ids = id.to_array_of_size(num_rows)?;
+ columns.insert(0, grouping_ids);
+ }
+ Ok(columns)
+}
+
impl AggregateStream {
/// Create a new AggregateStream
pub fn new(
@@ -350,6 +367,9 @@ impl AggregateStream {
let timer = this.baseline_metrics.elapsed_compute().timer();
let result =
finalize_aggregation(&mut this.accumulators, &this.mode)
+ .and_then(|columns| {
+ prepend_grouping_id_column(columns, None)
+ })
.and_then(|columns| {
RecordBatch::try_new(
Arc::clone(&this.schema),
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index cb2eb515ca..6616f77a5b 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -1223,6 +1223,7 @@ message AggregateExecNode {
repeated bool groups = 9;
repeated MaybeFilter filter_expr = 10;
AggLimit limit = 11;
+ bool has_grouping_set = 12;
}
message GlobalLimitExecNode {
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index 9b25b7b803..cf3dcfe01b 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -134,6 +134,9 @@ impl serde::Serialize for AggregateExecNode {
if self.limit.is_some() {
len += 1;
}
+ if self.has_grouping_set {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.AggregateExecNode", len)?;
if !self.group_expr.is_empty() {
struct_ser.serialize_field("groupExpr", &self.group_expr)?;
@@ -170,6 +173,9 @@ impl serde::Serialize for AggregateExecNode {
if let Some(v) = self.limit.as_ref() {
struct_ser.serialize_field("limit", v)?;
}
+ if self.has_grouping_set {
+ struct_ser.serialize_field("hasGroupingSet", &self.has_grouping_set)?;
+ }
struct_ser.end()
}
}
@@ -198,6 +204,8 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {
"filter_expr",
"filterExpr",
"limit",
+ "has_grouping_set",
+ "hasGroupingSet",
];
#[allow(clippy::enum_variant_names)]
@@ -213,6 +221,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {
Groups,
FilterExpr,
Limit,
+ HasGroupingSet,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -245,6 +254,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {
"groups" => Ok(GeneratedField::Groups),
"filterExpr" | "filter_expr" => Ok(GeneratedField::FilterExpr),
"limit" => Ok(GeneratedField::Limit),
+ "hasGroupingSet" | "has_grouping_set" => Ok(GeneratedField::HasGroupingSet),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -275,6 +285,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {
let mut groups__ = None;
let mut filter_expr__ = None;
let mut limit__ = None;
+ let mut has_grouping_set__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::GroupExpr => {
@@ -343,6 +354,12 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {
}
limit__ = map_.next_value()?;
}
+ GeneratedField::HasGroupingSet => {
+ if has_grouping_set__.is_some() {
+ return Err(serde::de::Error::duplicate_field("hasGroupingSet"));
+ }
+ has_grouping_set__ = Some(map_.next_value()?);
+ }
}
}
Ok(AggregateExecNode {
@@ -357,6 +374,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {
groups: groups__.unwrap_or_default(),
filter_expr: filter_expr__.unwrap_or_default(),
limit: limit__,
+ has_grouping_set: has_grouping_set__.unwrap_or_default(),
})
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 4d16f50e14..885b61001c 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1839,6 +1839,8 @@ pub struct AggregateExecNode {
pub filter_expr: ::prost::alloc::vec::Vec<MaybeFilter>,
#[prost(message, optional, tag = "11")]
pub limit: ::core::option::Option<AggLimit>,
+ #[prost(bool, tag = "12")]
+ pub has_grouping_set: bool,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct GlobalLimitExecNode {
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 6f275d2a22..836972a783 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -1010,6 +1010,8 @@ impl protobuf::PhysicalPlanNode {
vec![]
};
+ let has_grouping_set = hash_agg.has_grouping_set;
+
let input_schema = hash_agg.input_schema.as_ref().ok_or_else(|| {
internal_datafusion_err!("input_schema in AggregateNode is missing.")
})?;
@@ -1107,7 +1109,7 @@ impl protobuf::PhysicalPlanNode {
let agg = AggregateExec::try_new(
agg_mode,
- PhysicalGroupBy::new(group_expr, null_expr, groups),
+ PhysicalGroupBy::new(group_expr, null_expr, groups, has_grouping_set),
physical_aggr_expr,
physical_filter_expr,
input,
@@ -2540,6 +2542,7 @@ impl protobuf::PhysicalPlanNode {
null_expr,
groups,
limit,
+ has_grouping_set: exec.group_expr().has_grouping_set(),
},
))),
})
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index 19c2f437d8..1f6497a0c5 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -1278,7 +1278,7 @@ fn roundtrip_scalar_udf_extension_codec() -> Result<()> {
let aggregate = Arc::new(AggregateExec::try_new(
AggregateMode::Final,
- PhysicalGroupBy::new(vec![], vec![], vec![]),
+ PhysicalGroupBy::new(vec![], vec![], vec![], false),
vec![aggr_expr],
vec![None],
window,
@@ -1396,7 +1396,7 @@ fn roundtrip_aggregate_udf_extension_codec() -> Result<()> {
let aggregate = Arc::new(AggregateExec::try_new(
AggregateMode::Final,
- PhysicalGroupBy::new(vec![], vec![], vec![]),
+ PhysicalGroupBy::new(vec![], vec![], vec![], false),
vec![aggr_expr],
vec![None],
window,
diff --git a/datafusion/sqllogictest/test_files/grouping.slt b/datafusion/sqllogictest/test_files/grouping.slt
index 64d040d012..3d38576bdb 100644
--- a/datafusion/sqllogictest/test_files/grouping.slt
+++ b/datafusion/sqllogictest/test_files/grouping.slt
@@ -212,3 +212,15 @@ select c1, grouping(c1, c2) from test group by CUBE(c1);
statement error zero arguments
select c1, grouping() from test group by CUBE(c1);
+
+# grouping_sets_with_empty_set
+query I
+SELECT COUNT(*) FROM test GROUP BY GROUPING SETS (());
+----
+2
+
+# grouping_sets_with_empty_set
+query I
+SELECT SUM(v1) FROM generate_series(10) AS t1(v1) GROUP BY GROUPING SETS(())
+----
+55