Apache DataFusion SQL Query Engine
fix(substrait): schema errors for Aggregates with no groupings (#17909)
Victor Barua1b001a14a1dd
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rsindex c919bd0389..fce074cd51 100644--- a/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs+++ b/datafusion/substrait/src/logical_plan/consumer/rel/aggregate_rel.rs@@ -40,6 +40,7 @@ pub async fn from_aggregate_rel(let mut aggr_exprs = vec![];match agg.groupings.len() {+ 0 => {}1 => {group_exprs.extend_from_slice(&from_substrait_grouping(diff --git a/datafusion/substrait/tests/cases/aggregation_tests.rs b/datafusion/substrait/tests/cases/aggregation_tests.rsnew file mode 100644index 0000000000..815550bca5--- /dev/null+++ b/datafusion/substrait/tests/cases/aggregation_tests.rs@@ -0,0 +1,71 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Tests to verify aggregation relation handling in Substrait++#[cfg(test)]+mod tests {+ use crate::utils::test::{add_plan_schemas_to_ctx, read_json};+ use datafusion::common::Result;+ use datafusion::dataframe::DataFrame;+ use datafusion::prelude::SessionContext;+ use datafusion_substrait::logical_plan::consumer::from_substrait_plan;+ use insta::assert_snapshot;++ #[tokio::test]+ async fn no_grouping_set() -> Result<()> {+ let proto_plan =+ read_json("tests/testdata/test_plans/aggregate_groupings/no_groupings.json");+ let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;+ let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;++ assert_snapshot!(+ plan,+ @r#"+ Aggregate: groupBy=[[]], aggr=[[sum(c0) AS summation]]+ EmptyRelation: rows=0+ "#+ );++ // Trigger execution to ensure plan validity+ DataFrame::new(ctx.state(), plan).show().await?;++ Ok(())+ }++ #[tokio::test]+ async fn one_grouping_set() -> Result<()> {+ let proto_plan = read_json(+ "tests/testdata/test_plans/aggregate_groupings/single_grouping.json",+ );+ let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;+ let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;++ assert_snapshot!(+ plan,+ @r#"+ Aggregate: groupBy=[[c0]], aggr=[[sum(c0) AS summation]]+ EmptyRelation: rows=0+ "#+ );++ // Trigger execution to ensure plan validity+ DataFrame::new(ctx.state(), plan).show().await?;++ Ok(())+ }+}diff --git a/datafusion/substrait/tests/cases/mod.rs b/datafusion/substrait/tests/cases/mod.rsindex 9e69bb4edd..0870c56cd3 100644--- a/datafusion/substrait/tests/cases/mod.rs+++ b/datafusion/substrait/tests/cases/mod.rs@@ -15,6 +15,7 @@// specific language governing permissions and limitations// under the License.+mod aggregation_tests;mod builtin_expr_semantics_tests;mod consumer_integration;mod emit_kind_tests;diff --git a/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/no_groupings.json b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/no_groupings.jsonnew file mode 100644index 0000000000..9305aa6461--- /dev/null+++ b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/no_groupings.json@@ -0,0 +1,92 @@+{+ "extensionUris": [+ {+ "extensionUriAnchor": 1,+ "uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"+ }+ ],+ "extensions": [+ {+ "extensionFunction": {+ "extensionUriReference": 1,+ "functionAnchor": 1,+ "name": "sum:i8"+ }+ }+ ],+ "relations": [+ {+ "root": {+ "input": {+ "aggregate": {+ "common": {+ "direct": {}+ },+ "input": {+ "read": {+ "baseSchema": {+ "names": [+ "c0",+ "c1"+ ],+ "struct": {+ "nullability": "NULLABILITY_REQUIRED",+ "types": [+ {+ "i8": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i8": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ]+ }+ },+ "common": {+ "direct": {}+ },+ "virtualTable": {}+ }+ },+ "measures": [+ {+ "measure": {+ "arguments": [+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "rootReference": {}+ }+ }+ }+ ],+ "functionReference": 1,+ "invocation": "AGGREGATION_INVOCATION_ALL",+ "outputType": {+ "i8": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT"+ }+ }+ ]+ }+ },+ "names": [+ "summation"+ ]+ }+ }+ ],+ "version": {+ "minorNumber": 29,+ "producer": "substrait-go v4.2.0"+ }+}\ No newline at end of filediff --git a/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/single_grouping.json b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/single_grouping.jsonnew file mode 100644index 0000000000..9535596a1e--- /dev/null+++ b/datafusion/substrait/tests/testdata/test_plans/aggregate_groupings/single_grouping.json@@ -0,0 +1,109 @@+{+ "extensionUris": [+ {+ "extensionUriAnchor": 1,+ "uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"+ }+ ],+ "extensions": [+ {+ "extensionFunction": {+ "extensionUriReference": 1,+ "functionAnchor": 1,+ "name": "sum:i8"+ }+ }+ ],+ "relations": [+ {+ "root": {+ "input": {+ "aggregate": {+ "common": {+ "direct": {}+ },+ "input": {+ "read": {+ "baseSchema": {+ "names": [+ "c0",+ "c1"+ ],+ "struct": {+ "nullability": "NULLABILITY_REQUIRED",+ "types": [+ {+ "i8": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i8": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ]+ }+ },+ "common": {+ "direct": {}+ },+ "virtualTable": {}+ }+ },+ "groupingExpressions": [+ {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "rootReference": {}+ }+ }+ ],+ "groupings": [+ {+ "expressionReferences": [0]+ }++ ],+ "measures": [+ {+ "measure": {+ "arguments": [+ {+ "value": {+ "selection": {+ "directReference": {+ "structField": {}+ },+ "rootReference": {}+ }+ }+ }+ ],+ "functionReference": 1,+ "invocation": "AGGREGATION_INVOCATION_ALL",+ "outputType": {+ "i8": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT"+ }+ }+ ]+ }+ },+ "names": [+ "c0",+ "summation"+ ]+ }+ }+ ],+ "version": {+ "minorNumber": 29,+ "producer": "substrait-go v4.2.0"+ }+}\ No newline at end of file