Skip to content

Apache DataFusion SQL Query Engine

fix(substrait): schema errors for Aggregates with no groupings (#17909)

Victor Barua1b001a14a1dd
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs
index c919bd0389..fce074cd51 100644
--- a/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs
@@ -40,6 +40,7 @@ pub async fn from_aggregate_rel(
let mut aggr_exprs = vec![];
match agg.groupings.len() {
+ 0 => {}
1 => {
group_exprs.extend_from_slice(
&from_substrait_grouping(
diff --git a/datafusion/substrait/tests/cases/aggregation_tests.rs b/datafusion/substrait/tests/cases/aggregation_tests.rs
new file mode 100644
index 0000000000..815550bca5
--- /dev/null
+++ b/datafusion/substrait/tests/cases/aggregation_tests.rs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Tests to verify aggregation relation handling in Substrait
+
+#[cfg(test)]
+mod tests {
+ use crate::utils::test::{add_plan_schemas_to_ctx, read_json};
+ use datafusion::common::Result;
+ use datafusion::dataframe::DataFrame;
+ use datafusion::prelude::SessionContext;
+ use datafusion_substrait::logical_plan::consumer::from_substrait_plan;
+ use insta::assert_snapshot;
+
+ #[tokio::test]
+ async fn no_grouping_set() -> Result<()> {
+ let proto_plan =
+ read_json("tests/testdata/test_plans/aggregate_groupings/no_groupings.json");
+ let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
+ let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
+
+ assert_snapshot!(
+ plan,
+ @r#"
+ Aggregate: groupBy=[[]], aggr=[[sum(c0) AS summation]]
+ EmptyRelation: rows=0
+ "#
+ );
+
+ // Trigger execution to ensure plan validity
+ DataFrame::new(ctx.state(), plan).show().await?;
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn one_grouping_set() -> Result<()> {
+ let proto_plan = read_json(
+ "tests/testdata/test_plans/aggregate_groupings/single_grouping.json",
+ );
+ let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
+ let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
+
+ assert_snapshot!(
+ plan,
+ @r#"
+ Aggregate: groupBy=[[c0]], aggr=[[sum(c0) AS summation]]
+ EmptyRelation: rows=0
+ "#
+ );
+
+ // Trigger execution to ensure plan validity
+ DataFrame::new(ctx.state(), plan).show().await?;
+
+ Ok(())
+ }
+}
diff --git a/datafusion/substrait/tests/cases/mod.rs b/datafusion/substrait/tests/cases/mod.rs
index 9e69bb4edd..0870c56cd3 100644
--- a/datafusion/substrait/tests/cases/mod.rs
+++ b/datafusion/substrait/tests/cases/mod.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+mod aggregation_tests;
mod builtin_expr_semantics_tests;
mod consumer_integration;
mod emit_kind_tests;
diff --git a/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/no_groupings.json b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/no_groupings.json
new file mode 100644
index 0000000000..9305aa6461
--- /dev/null
+++ b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/no_groupings.json
@@ -0,0 +1,92 @@
+{
+ "extensionUris": [
+ {
+ "extensionUriAnchor": 1,
+ "uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"
+ }
+ ],
+ "extensions": [
+ {
+ "extensionFunction": {
+ "extensionUriReference": 1,
+ "functionAnchor": 1,
+ "name": "sum:i8"
+ }
+ }
+ ],
+ "relations": [
+ {
+ "root": {
+ "input": {
+ "aggregate": {
+ "common": {
+ "direct": {}
+ },
+ "input": {
+ "read": {
+ "baseSchema": {
+ "names": [
+ "c0",
+ "c1"
+ ],
+ "struct": {
+ "nullability": "NULLABILITY_REQUIRED",
+ "types": [
+ {
+ "i8": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ {
+ "i8": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ }
+ ]
+ }
+ },
+ "common": {
+ "direct": {}
+ },
+ "virtualTable": {}
+ }
+ },
+ "measures": [
+ {
+ "measure": {
+ "arguments": [
+ {
+ "value": {
+ "selection": {
+ "directReference": {
+ "structField": {}
+ },
+ "rootReference": {}
+ }
+ }
+ }
+ ],
+ "functionReference": 1,
+ "invocation": "AGGREGATION_INVOCATION_ALL",
+ "outputType": {
+ "i8": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT"
+ }
+ }
+ ]
+ }
+ },
+ "names": [
+ "summation"
+ ]
+ }
+ }
+ ],
+ "version": {
+ "minorNumber": 29,
+ "producer": "substrait-go v4.2.0"
+ }
+}
\ No newline at end of file
diff --git a/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/single_grouping.json b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/single_grouping.json
new file mode 100644
index 0000000000..9535596a1e
--- /dev/null
+++ b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/single_grouping.json
@@ -0,0 +1,109 @@
+{
+ "extensionUris": [
+ {
+ "extensionUriAnchor": 1,
+ "uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"
+ }
+ ],
+ "extensions": [
+ {
+ "extensionFunction": {
+ "extensionUriReference": 1,
+ "functionAnchor": 1,
+ "name": "sum:i8"
+ }
+ }
+ ],
+ "relations": [
+ {
+ "root": {
+ "input": {
+ "aggregate": {
+ "common": {
+ "direct": {}
+ },
+ "input": {
+ "read": {
+ "baseSchema": {
+ "names": [
+ "c0",
+ "c1"
+ ],
+ "struct": {
+ "nullability": "NULLABILITY_REQUIRED",
+ "types": [
+ {
+ "i8": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ {
+ "i8": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ }
+ ]
+ }
+ },
+ "common": {
+ "direct": {}
+ },
+ "virtualTable": {}
+ }
+ },
+ "groupingExpressions": [
+ {
+ "selection": {
+ "directReference": {
+ "structField": {}
+ },
+ "rootReference": {}
+ }
+ }
+ ],
+ "groupings": [
+ {
+ "expressionReferences": [0]
+ }
+
+ ],
+ "measures": [
+ {
+ "measure": {
+ "arguments": [
+ {
+ "value": {
+ "selection": {
+ "directReference": {
+ "structField": {}
+ },
+ "rootReference": {}
+ }
+ }
+ }
+ ],
+ "functionReference": 1,
+ "invocation": "AGGREGATION_INVOCATION_ALL",
+ "outputType": {
+ "i8": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT"
+ }
+ }
+ ]
+ }
+ },
+ "names": [
+ "c0",
+ "summation"
+ ]
+ }
+ }
+ ],
+ "version": {
+ "minorNumber": 29,
+ "producer": "substrait-go v4.2.0"
+ }
+}
\ No newline at end of file