Skip to content

Apache DataFusion SQL Query Engine

Allow a non-distinct count alongside the single distinct aggregate in SingleDistinctToGroupBy (#24859)

Adrian Garcia Badaracco4f0faac25d21
diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rs
index 8f7e9cc6cf..458b39c969 100644
--- a/datafusion/expr/src/udaf.rs
+++ b/datafusion/expr/src/udaf.rs
@@ -253,6 +253,16 @@ impl AggregateUDF {
self.inner.groups_accumulator_supported(args)
}
+ /// See [`AggregateUDFImpl::groups_accumulator_supported_for_types`] for more details.
+ pub fn groups_accumulator_supported_for_types(
+ &self,
+ arg_types: &[DataType],
+ is_distinct: bool,
+ ) -> Option<bool> {
+ self.inner
+ .groups_accumulator_supported_for_types(arg_types, is_distinct)
+ }
+
/// See [`AggregateUDFImpl::create_groups_accumulator`] for more details.
pub fn create_groups_accumulator(
&self,
@@ -617,6 +627,32 @@ pub trait AggregateUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {
false
}
+ /// The same question as [`Self::groups_accumulator_supported`], asked with
+ /// only the information a logical plan carries.
+ ///
+ /// Physical planning has an [`AccumulatorArgs`] to ask with; an optimizer
+ /// rule does not. This is how a logical caller learns whether a call would
+ /// get a specialized [`GroupsAccumulator`] or fall back to one boxed
+ /// [`Accumulator`] per group in `GroupsAccumulatorAdapter`, whose per-group
+ /// state can be orders of magnitude larger.
+ ///
+ /// An implementation whose answer is decided by the argument types and
+ /// `DISTINCT` alone should override this and have
+ /// [`Self::groups_accumulator_supported`] call it, so the two cannot
+ /// disagree. One whose answer needs more than that should leave the `None`
+ /// default.
+ ///
+ /// `None` means unanswered, not "no": a caller must not read it as either
+ /// `Some(true)` or `Some(false)`, since both readings are wrong for some
+ /// implementation that returns it.
+ fn groups_accumulator_supported_for_types(
+ &self,
+ _arg_types: &[DataType],
+ _is_distinct: bool,
+ ) -> Option<bool> {
+ None
+ }
+
/// Return a specialized [`GroupsAccumulator`] that manages state
/// for all groups.
///
@@ -1552,6 +1588,15 @@ impl AggregateUDFImpl for AliasedAggregateUDFImpl {
self.inner.groups_accumulator_supported(args)
}
+ fn groups_accumulator_supported_for_types(
+ &self,
+ arg_types: &[DataType],
+ is_distinct: bool,
+ ) -> Option<bool> {
+ self.inner
+ .groups_accumulator_supported_for_types(arg_types, is_distinct)
+ }
+
fn create_groups_accumulator(
&self,
args: AccumulatorArgs,
diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rs
index 4c95ea4318..2086c4c41b 100644
--- a/datafusion/functions-aggregate/src/count.rs
+++ b/datafusion/functions-aggregate/src/count.rs
@@ -346,14 +346,31 @@ impl AggregateUDFImpl for Count {
}
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
- if args.exprs.len() != 1 {
- return false;
+ // The answer depends on nothing but the argument types and `DISTINCT`,
+ // so defer to the logical form and keep one list of supported types.
+ let arg_types = args
+ .expr_fields
+ .iter()
+ .map(|field| field.data_type().clone())
+ .collect::<Vec<_>>();
+ self.groups_accumulator_supported_for_types(&arg_types, args.is_distinct)
+ .unwrap_or(false)
+ }
+
+ fn groups_accumulator_supported_for_types(
+ &self,
+ arg_types: &[DataType],
+ is_distinct: bool,
+ ) -> Option<bool> {
+ if arg_types.len() != 1 {
+ return Some(false);
}
- if !args.is_distinct {
- return true;
+ if !is_distinct {
+ return Some(true);
}
- matches!(
- args.expr_fields[0].data_type(),
+ // Keep in step with `create_distinct_count_groups_accumulator`.
+ Some(matches!(
+ arg_types[0],
DataType::Int8
| DataType::Int16
| DataType::Int32
@@ -362,7 +379,7 @@ impl AggregateUDFImpl for Count {
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
- )
+ ))
}
fn create_groups_accumulator(
diff --git a/datafusion/optimizer/src/single_distinct_to_groupby.rs b/datafusion/optimizer/src/single_distinct_to_groupby.rs
index 00c8fab228..f663003a9e 100644
--- a/datafusion/optimizer/src/single_distinct_to_groupby.rs
+++ b/datafusion/optimizer/src/single_distinct_to_groupby.rs
@@ -23,14 +23,18 @@ use crate::optimizer::ApplyOrder;
use crate::{OptimizerConfig, OptimizerRule};
use datafusion_common::{
- DataFusionError, HashSet, Result, assert_eq_or_internal_err, tree_node::Transformed,
+ DFSchema, DataFusionError, HashSet, Result, assert_eq_or_internal_err,
+ tree_node::Transformed,
};
use datafusion_expr::builder::project;
use datafusion_expr::expr::AggregateFunctionParams;
+use datafusion_expr::expr_schema::ExprSchemable;
use datafusion_expr::{
- Expr, col,
+ AggregateUDF, Expr, col,
expr::AggregateFunction,
+ lit,
logical_plan::{Aggregate, LogicalPlan},
+ when,
};
/// single distinct to group by optimizer rule
@@ -49,6 +53,34 @@ use datafusion_expr::{
/// )
/// GROUP BY a
/// ```
+///
+/// A non-distinct `count` may also appear alongside the distinct aggregate. It
+/// is the one supported function whose outer phase is a different function: the
+/// inner group-by counts rows per `(group, distinct value)` pair, and the outer
+/// phase sums those partial counts.
+///
+/// ```text
+/// Before:
+/// SELECT a, count(*), count(DISTINCT b)
+/// FROM t
+/// GROUP BY a
+///
+/// After:
+/// SELECT a,
+/// CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE 0 END,
+/// count(alias1)
+/// FROM (
+/// SELECT a, b as alias1, count(*) as alias2
+/// FROM t
+/// GROUP BY a, b
+/// )
+/// GROUP BY a
+/// ```
+///
+/// The `CASE` restores the 0 that `count` reports over an empty input, where a
+/// `sum` of no rows is NULL.
+///
+/// See `rewrite_pays_for_count` for when the `count` is allowed.
#[derive(Default, Debug)]
pub struct SingleDistinctToGroupBy {}
@@ -61,10 +93,43 @@ impl SingleDistinctToGroupBy {
}
}
+/// The pair of functions used to compute a non-distinct `count` in two phases:
+/// `count` identifies the aggregates that need the treatment, `sum` combines
+/// the partial counts the inner group-by produces.
+///
+/// Both are resolved from the session's function registry, so a plan built
+/// without one keeps the previous behaviour of bailing out on `count`.
+struct CountRollup {
+ count: Arc<AggregateUDF>,
+ sum: Arc<AggregateUDF>,
+}
+
+impl CountRollup {
+ fn try_new(config: &dyn OptimizerConfig) -> Option<Self> {
+ let registry = config.function_registry()?;
+ Some(Self {
+ count: registry.udaf("count").ok()?,
+ sum: registry.udaf("sum").ok()?,
+ })
+ }
+
+ /// Whether `func` is the registry's `count`, and so decomposes into
+ /// `sum` over per-partition counts.
+ fn is_count(&self, func: &AggregateUDF) -> bool {
+ self.count.as_ref() == func
+ }
+}
+
/// Check whether all aggregate exprs are distinct on a single field.
-fn is_single_distinct_agg(aggr_expr: &[Expr]) -> Result<bool> {
+fn is_single_distinct_agg(
+ aggr_expr: &[Expr],
+ input_schema: &DFSchema,
+ count_rollup: Option<&CountRollup>,
+) -> Result<bool> {
let mut fields_set = HashSet::new();
let mut aggregate_count = 0;
+ let mut distinct_aggs = vec![];
+ let mut has_count_rollup = false;
for expr in aggr_expr {
if let Expr::AggregateFunction(AggregateFunction {
func,
@@ -86,6 +151,9 @@ fn is_single_distinct_agg(aggr_expr: &[Expr]) -> Result<bool> {
for e in args {
fields_set.insert(e);
}
+ distinct_aggs.push((func, args.as_slice()));
+ } else if count_rollup.is_some_and(|rollup| rollup.is_count(func)) {
+ has_count_rollup = true;
} else if func.name() != "sum"
&& func.name().to_lowercase() != "min"
&& func.name().to_lowercase() != "max"
@@ -96,7 +164,50 @@ fn is_single_distinct_agg(aggr_expr: &[Expr]) -> Result<bool> {
return Ok(false);
}
}
- Ok(aggregate_count == aggr_expr.len() && fields_set.len() == 1)
+ if aggregate_count != aggr_expr.len() || fields_set.len() != 1 {
+ return Ok(false);
+ }
+ if has_count_rollup && !rewrite_pays_for_count(&distinct_aggs, input_schema)? {
+ return Ok(false);
+ }
+ Ok(true)
+}
+
+/// Whether the rewrite is worth extending to a plan that only qualifies because
+/// of the non-distinct `count`.
+///
+/// The rewrite is not free: every other aggregate moves down to the inner group
+/// by, which has a row per `(group, distinct value)` pair rather than per group,
+/// and keeps its state at that finer grain. What pays for that is taking the
+/// distinct aggregate off `GroupsAccumulatorAdapter` and its one boxed
+/// accumulator per group. An aggregate that already has a specialized
+/// `GroupsAccumulator` never went near the adapter, so there is nothing to buy
+/// and only the inner group by to pay for.
+///
+/// `Some(false)` is therefore the only answer that permits the rewrite. `None`,
+/// the default for almost every function, is not evidence that it pays.
+///
+/// This is a proxy: what actually decides the outcome is the cost per distinct
+/// value on each side, which this rule cannot see. It is deliberately
+/// conservative in the direction that leaves a plan alone. The pre-existing
+/// tolerance of a non-distinct `sum`, `min` or `max` is not gated, since
+/// narrowing it would change plans that have always been rewritten.
+///
+/// Measured in <https://github.com/apache/datafusion/pull/24859>.
+fn rewrite_pays_for_count(
+ distinct_aggs: &[(&Arc<AggregateUDF>, &[Expr])],
+ input_schema: &DFSchema,
+) -> Result<bool> {
+ for (func, args) in distinct_aggs {
+ let arg_types = args
+ .iter()
+ .map(|arg| arg.get_type(input_schema))
+ .collect::<Result<Vec<_>>>()?;
+ if func.groups_accumulator_supported_for_types(&arg_types, true) == Some(false) {
+ return Ok(true);
+ }
+ }
+ Ok(false)
}
/// Check if the first expr is [Expr::GroupingSet].
@@ -120,8 +231,12 @@ impl OptimizerRule for SingleDistinctToGroupBy {
fn rewrite(
&self,
plan: LogicalPlan,
- _config: &dyn OptimizerConfig,
+ config: &dyn OptimizerConfig,
) -> Result<Transformed<LogicalPlan>, DataFusionError> {
+ if !matches!(plan, LogicalPlan::Aggregate(_)) {
+ return Ok(Transformed::no(plan));
+ }
+ let count_rollup = CountRollup::try_new(config);
match plan {
LogicalPlan::Aggregate(Aggregate {
input,
@@ -129,8 +244,11 @@ impl OptimizerRule for SingleDistinctToGroupBy {
schema,
group_expr,
..
- }) if is_single_distinct_agg(&aggr_expr)?
- && !contains_grouping_set(&group_expr) =>
+ }) if is_single_distinct_agg(
+ &aggr_expr,
+ input.schema(),
+ count_rollup.as_ref(),
+ )? && !contains_grouping_set(&group_expr) =>
{
let group_size = group_expr.len();
// alias all original group_by exprs
@@ -177,7 +295,11 @@ impl OptimizerRule for SingleDistinctToGroupBy {
let mut index = 1;
let mut group_fields_set = HashSet::new();
let mut inner_aggr_exprs = vec![];
- let outer_aggr_exprs = aggr_expr
+ // Each aggregate yields the expression the outer `Aggregate`
+ // computes and the expression the projection above it selects.
+ // They differ only for `count`, whose projection restores the
+ // zero that `sum` reports as NULL over an empty input.
+ let (outer_aggr_exprs, outer_proj_exprs): (Vec<Expr>, Vec<Expr>) = aggr_expr
.into_iter()
.map(|aggr_expr| match aggr_expr {
Expr::AggregateFunction(AggregateFunction {
@@ -204,18 +326,29 @@ impl OptimizerRule for SingleDistinctToGroupBy {
inner_group_exprs
.push(arg.alias(SINGLE_DISTINCT_ALIAS));
}
- Ok(Expr::AggregateFunction(AggregateFunction::new_udf(
- func,
- vec![col(SINGLE_DISTINCT_ALIAS)],
- false, // intentional to remove distinct here
- filter,
- order_by,
- null_treatment,
- )))
+ let outer =
+ Expr::AggregateFunction(AggregateFunction::new_udf(
+ func,
+ vec![col(SINGLE_DISTINCT_ALIAS)],
+ false, // intentional to remove distinct here
+ filter,
+ order_by,
+ null_treatment,
+ ));
+ Ok((outer.clone(), outer))
// if the aggregate function is not distinct, we need to rewrite it like two phase aggregation
} else {
index += 1;
let alias_str = format!("alias{index}");
+ // `count`'s outer phase is `sum`, over the
+ // partial counts the inner group-by produced.
+ let rollup = count_rollup
+ .as_ref()
+ .filter(|rollup| rollup.is_count(&func));
+ let outer_func = match rollup {
+ Some(rollup) => Arc::clone(&rollup.sum),
+ None => Arc::clone(&func),
+ };
inner_aggr_exprs.push(
Expr::AggregateFunction(AggregateFunction::new_udf(
Arc::clone(&func),
@@ -227,19 +360,32 @@ impl OptimizerRule for SingleDistinctToGroupBy {
))
.alias(&alias_str),
);
- Ok(Expr::AggregateFunction(AggregateFunction::new_udf(
- func,
- vec![col(&alias_str)],
- false,
- None,
- vec![],
- None,
- )))
+ let outer =
+ Expr::AggregateFunction(AggregateFunction::new_udf(
+ outer_func,
+ vec![col(&alias_str)],
+ false,
+ None,
+ vec![],
+ None,
+ ));
+ if rollup.is_none() {
+ return Ok((outer.clone(), outer));
+ }
+ // `sum` of no rows is NULL where `count` is 0.
+ // Restoring the 0 also keeps the column
+ // non-nullable, as `count` had it.
+ let proj =
+ when(outer.clone().is_not_null(), outer.clone())
+ .otherwise(lit(0_i64))?;
+ Ok((outer, proj))
}
}
- _ => Ok(aggr_expr),
+ _ => Ok((aggr_expr.clone(), aggr_expr)),
})
- .collect::<Result<Vec<_>>>()?;
+ .collect::<Result<Vec<_>>>()?
+ .into_iter()
+ .unzip();
// construct the inner AggrPlan
let inner_agg = LogicalPlan::Aggregate(Aggregate::try_new(
@@ -265,13 +411,11 @@ impl OptimizerRule for SingleDistinctToGroupBy {
}
None => group_expr,
})
- .chain(outer_aggr_exprs.iter().cloned().enumerate().map(
- |(idx, expr)| {
- let idx = idx + group_size;
- let (qualifier, field) = schema.qualified_field(idx);
- expr.alias_qualified(qualifier.cloned(), field.name())
- },
- ))
+ .chain(outer_proj_exprs.into_iter().enumerate().map(|(idx, expr)| {
+ let idx = idx + group_size;
+ let (qualifier, field) = schema.qualified_field(idx);
+ expr.alias_qualified(qualifier.cloned(), field.name())
+ }))
.collect();
let outer_aggr = LogicalPlan::Aggregate(Aggregate::try_new(
@@ -291,9 +435,20 @@ mod tests {
use super::*;
use crate::assert_optimized_plan_eq_display_indent_snapshot;
use crate::test::*;
+ use crate::{Optimizer, OptimizerContext};
+ use arrow::datatypes::{DataType, Field, Schema};
+ use chrono::{DateTime, Utc};
+ use datafusion_common::alias::AliasGenerator;
+ use datafusion_common::config::ConfigOptions;
use datafusion_expr::ExprFunctionExt;
use datafusion_expr::expr::GroupingSet;
- use datafusion_expr::{lit, logical_plan::builder::LogicalPlanBuilder};
+ use datafusion_expr::function::AccumulatorArgs;
+ use datafusion_expr::registry::{FunctionRegistry, MemoryFunctionRegistry};
+ use datafusion_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility};
+ use datafusion_expr::{
+ lit,
+ logical_plan::builder::{LogicalPlanBuilder, table_scan},
+ };
use datafusion_functions_aggregate::count::count_udaf;
use datafusion_functions_aggregate::expr_fn::{count, count_distinct, max, min, sum};
use datafusion_functions_aggregate::min_max::max_udaf;
@@ -310,17 +465,143 @@ mod tests {
))
}
+ fn sum_distinct(expr: Expr) -> Expr {
+ Expr::AggregateFunction(AggregateFunction::new_udf(
+ sum_udaf(),
+ vec![expr],
+ true,
+ None,
+ vec![],
+ None,
+ ))
+ }
+
+ /// An aggregate that leaves
+ /// [`AggregateUDFImpl::groups_accumulator_supported_for_types`] at its
+ /// default, which is what almost every function in the wild does.
+ #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+ struct SilentUdaf {
+ signature: Signature,
+ }
+
+ impl SilentUdaf {
+ fn new() -> Self {
+ Self {
+ signature: Signature::any(1, Volatility::Immutable),
+ }
+ }
+ }
+
+ impl AggregateUDFImpl for SilentUdaf {
+ fn name(&self) -> &str {
+ "silent"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::UInt32)
+ }
+
+ fn accumulator(
+ &self,
+ _acc_args: AccumulatorArgs,
+ ) -> Result<Box<dyn Accumulator>> {
+ unimplemented!("not needed for this test")
+ }
+ }
+
+ fn silent_distinct(expr: Expr) -> Expr {
+ Expr::AggregateFunction(AggregateFunction::new_udf(
+ Arc::new(AggregateUDF::from(SilentUdaf::new())),
+ vec![expr],
+ true,
+ None,
+ vec![],
+ None,
+ ))
+ }
+
+ fn count_star() -> Expr {
+ Expr::AggregateFunction(AggregateFunction::new_udf(
+ count_udaf(),
+ vec![lit(1_i64)],
+ false,
+ None,
+ vec![],
+ None,
+ ))
+ }
+
+ /// `test` with a `Utf8` `b`, the column the distinct aggregate reads.
+ ///
+ /// `count(DISTINCT b)` over a string has no specialized
+ /// `GroupsAccumulator`, so it is the case a non-distinct `count` may join.
+ /// The `UInt32` `b` of [`test_table_scan`] is the case it may not.
+ fn test_table_scan_utf8_b() -> Result<LogicalPlan> {
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::UInt32, false),
+ Field::new("b", DataType::Utf8, false),
+ Field::new("c", DataType::UInt32, false),
+ ]);
+ table_scan(Some("test"), &schema, None)?.build()
+ }
+
+ /// An [`OptimizerConfig`] that resolves functions the way a session does.
+ /// The rule needs `count` and `sum` from the registry to rewrite a
+ /// non-distinct `count`.
+ #[derive(Debug)]
+ struct RegistryOptimizerContext {
+ inner: OptimizerContext,
+ registry: MemoryFunctionRegistry,
+ }
+
+ impl RegistryOptimizerContext {
+ fn new() -> Self {
+ let mut registry = MemoryFunctionRegistry::new();
+ registry.register_udaf(count_udaf()).unwrap();
+ registry.register_udaf(sum_udaf()).unwrap();
+ Self {
+ inner: OptimizerContext::new(),
+ registry,
+ }
+ }
+ }
+
+ impl OptimizerConfig for RegistryOptimizerContext {
+ fn query_execution_start_time(&self) -> Option<DateTime<Utc>> {
+ self.inner.query_execution_start_time()
+ }
+
+ fn alias_generator(&self) -> &Arc<AliasGenerator> {
+ self.inner.alias_generator()
+ }
+
+ fn options(&self) -> Arc<ConfigOptions> {
+ self.inner.options()
+ }
+
+ fn function_registry(&self) -> Option<&dyn FunctionRegistry> {
+ Some(&self.registry)
+ }
+ }
+
macro_rules! assert_optimized_plan_equal {
(
$plan:expr,
@ $expected:literal $(,)?
) => {{
- let rule: Arc<dyn crate::OptimizerRule + Send + Sync> = Arc::new(SingleDistinctToGroupBy::new());
- assert_optimized_plan_eq_display_indent_snapshot!(
- rule,
- $plan,
- @ $expected,
- )
+ let rule: Arc<dyn crate::OptimizerRule + Send + Sync> =
+ Arc::new(SingleDistinctToGroupBy::new());
+ let optimizer = Optimizer::with_rules(vec![rule]);
+ let optimized_plan = optimizer
+ .optimize($plan, &RegistryOptimizerContext::new(), |_, _| {})
+ .expect("failed to optimize plan");
+ insta::assert_snapshot!(optimized_plan.display_indent_schema(), @ $expected);
+
+ Ok::<(), DataFusionError>(())
}};
}
@@ -514,6 +795,30 @@ mod tests {
#[test]
fn distinct_and_common() -> Result<()> {
+ let table_scan = test_table_scan_utf8_b()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(
+ vec![col("a")],
+ vec![count_distinct(col("b")), count(col("c"))],
+ )?
+ .build()?;
+
+ // Should work: the non-distinct count becomes a sum of the counts the
+ // inner group-by produces per (test.a, test.b) pair
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: test.a, count(alias1) AS count(DISTINCT test.b), CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END AS count(test.c) [a:UInt32, count(DISTINCT test.b):Int64, count(test.c):Int64]
+ Aggregate: groupBy=[[test.a]], aggr=[[count(alias1), sum(alias2)]] [a:UInt32, count(alias1):Int64, sum(alias2):Int64;N]
+ Aggregate: groupBy=[[test.a, test.b AS alias1]], aggr=[[count(test.c) AS alias2]] [a:UInt32, alias1:Utf8, alias2:Int64]
+ TableScan: test [a:UInt32, b:Utf8, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn distinct_and_common_over_a_natively_supported_type() -> Result<()> {
let table_scan = test_table_scan()?;
let plan = LogicalPlanBuilder::from(table_scan)
@@ -523,7 +828,9 @@ mod tests {
)?
.build()?;
- // Do nothing
+ // Should not work: `count(DISTINCT b)` over a `UInt32` has its own
+ // `GroupsAccumulator`, so the rewrite would add an inner group-by
+ // without taking anything off `GroupsAccumulatorAdapter`
assert_optimized_plan_equal!(
plan,
@r"
@@ -533,6 +840,78 @@ mod tests {
)
}
+ #[test]
+ fn distinct_over_a_natively_supported_type_without_a_count() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(
+ vec![col("a")],
+ vec![count_distinct(col("b")), sum(col("c"))],
+ )?
+ .build()?;
+
+ // Should work: the gate covers only the `count` this change added, so a
+ // plan that already qualified through `sum`, `min` or `max` is rewritten
+ // exactly as before
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: test.a, count(alias1) AS count(DISTINCT test.b), sum(alias2) AS sum(test.c) [a:UInt32, count(DISTINCT test.b):Int64, sum(test.c):UInt64;N]
+ Aggregate: groupBy=[[test.a]], aggr=[[count(alias1), sum(alias2)]] [a:UInt32, count(alias1):Int64, sum(alias2):UInt64;N]
+ Aggregate: groupBy=[[test.a, test.b AS alias1]], aggr=[[sum(test.c) AS alias2]] [a:UInt32, alias1:UInt32, alias2:UInt64;N]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn distinct_over_a_function_with_no_opinion_is_not_rewritten() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(
+ vec![col("a")],
+ vec![silent_distinct(col("b")), count(col("c"))],
+ )?
+ .build()?;
+
+ // Should not work: `silent` leaves
+ // `groups_accumulator_supported_for_types` at its default, so it
+ // reports nothing about `GroupsAccumulatorAdapter`. Silence is not
+ // evidence that the rewrite pays, and the rule must not read it as
+ // permission
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Aggregate: groupBy=[[test.a]], aggr=[[silent(DISTINCT test.b), count(test.c)]] [a:UInt32, silent(DISTINCT test.b):UInt32;N, count(test.c):Int64]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn distinct_sum_and_count_is_not_rewritten() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(
+ vec![col("a")],
+ vec![sum_distinct(col("b")), count(col("c"))],
+ )?
+ .build()?;
+
+ // Should not work: `sum` reports nothing about
+ // `GroupsAccumulatorAdapter` either, so the rewrite is not known to pay
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Aggregate: groupBy=[[test.a]], aggr=[[sum(DISTINCT test.b), count(test.c)]] [a:UInt32, sum(DISTINCT test.b):UInt64;N, count(test.c):Int64]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
+
#[test]
fn group_by_with_expr() -> Result<()> {
let table_scan = test_table_scan().unwrap();
@@ -752,4 +1131,128 @@ mod tests {
"
)
}
+
+ #[test]
+ fn count_star_and_distinct_without_groupby() -> Result<()> {
+ let table_scan = test_table_scan_utf8_b()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(
+ Vec::<Expr>::new(),
+ vec![count_star(), count_distinct(col("b"))],
+ )?
+ .build()?;
+
+ // Should work. Without a group by the outer aggregate sees no rows at
+ // all for an empty input, so the projection has to turn the NULL that
+ // `sum` reports there back into the 0 `count` reports.
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END AS count(Int64(1)), count(alias1) AS count(DISTINCT test.b) [count(Int64(1)):Int64, count(DISTINCT test.b):Int64]
+ Aggregate: groupBy=[[]], aggr=[[sum(alias2), count(alias1)]] [sum(alias2):Int64;N, count(alias1):Int64]
+ Aggregate: groupBy=[[test.b AS alias1]], aggr=[[count(Int64(1)) AS alias2]] [alias1:Utf8, alias2:Int64]
+ TableScan: test [a:UInt32, b:Utf8, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn count_star_min_max_sum_and_distinct_with_groupby() -> Result<()> {
+ let table_scan = test_table_scan_utf8_b()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(
+ vec![col("a")],
+ vec![
+ count_star(),
+ count_distinct(col("b")),
+ min(col("c")),
+ max(col("c")),
+ sum(col("c")),
+ ],
+ )?
+ .build()?;
+
+ // Should work: this is the shape a `count(*)` alongside a
+ // `count(DISTINCT ..)`, a min, a max and a sum produces
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: test.a, CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END AS count(Int64(1)), count(alias1) AS count(DISTINCT test.b), min(alias3) AS min(test.c), max(alias4) AS max(test.c), sum(alias5) AS sum(test.c) [a:UInt32, count(Int64(1)):Int64, count(DISTINCT test.b):Int64, min(test.c):UInt32;N, max(test.c):UInt32;N, sum(test.c):UInt64;N]
+ Aggregate: groupBy=[[test.a]], aggr=[[sum(alias2), count(alias1), min(alias3), max(alias4), sum(alias5)]] [a:UInt32, sum(alias2):Int64;N, count(alias1):Int64, min(alias3):UInt32;N, max(alias4):UInt32;N, sum(alias5):UInt64;N]
+ Aggregate: groupBy=[[test.a, test.b AS alias1]], aggr=[[count(Int64(1)) AS alias2, min(test.c) AS alias3, max(test.c) AS alias4, sum(test.c) AS alias5]] [a:UInt32, alias1:Utf8, alias2:Int64, alias3:UInt32;N, alias4:UInt32;N, alias5:UInt64;N]
+ TableScan: test [a:UInt32, b:Utf8, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn count_with_filter_is_not_rewritten() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ // count(a) FILTER (WHERE a > 5)
+ let expr = count_udaf()
+ .call(vec![col("a")])
+ .filter(col("a").gt(lit(5)))
+ .build()?;
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(vec![col("c")], vec![expr, count_distinct(col("b"))])?
+ .build()?;
+
+ // Do nothing: the filter would have to be applied per input row, but
+ // the inner aggregate has already collapsed them
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Aggregate: groupBy=[[test.c]], aggr=[[count(test.a) FILTER (WHERE test.a > Int32(5)), count(DISTINCT test.b)]] [c:UInt32, count(test.a) FILTER (WHERE test.a > Int32(5)):Int64, count(DISTINCT test.b):Int64]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn count_with_order_by_is_not_rewritten() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ // count(a ORDER BY a)
+ let expr = count_udaf()
+ .call(vec![col("a")])
+ .order_by(vec![col("a").sort(true, false)])
+ .build()?;
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(vec![col("c")], vec![expr, count_distinct(col("b"))])?
+ .build()?;
+
+ // Do nothing
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Aggregate: groupBy=[[test.c]], aggr=[[count(test.a) ORDER BY [test.a ASC NULLS LAST], count(DISTINCT test.b)]] [c:UInt32, count(test.a) ORDER BY [test.a ASC NULLS LAST]:Int64, count(DISTINCT test.b):Int64]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
+
+ #[test]
+ fn count_without_function_registry_is_not_rewritten() -> Result<()> {
+ let table_scan = test_table_scan()?;
+
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .aggregate(vec![col("a")], vec![count_star(), count_distinct(col("b"))])?
+ .build()?;
+
+ // Do nothing: without a registry the rule cannot resolve the `sum` the
+ // outer phase needs
+ let rule: Arc<dyn OptimizerRule + Send + Sync> =
+ Arc::new(SingleDistinctToGroupBy::new());
+ assert_optimized_plan_eq_display_indent_snapshot!(
+ rule,
+ plan,
+ @r"
+ Aggregate: groupBy=[[test.a]], aggr=[[count(Int64(1)), count(DISTINCT test.b)]] [a:UInt32, count(Int64(1)):Int64, count(DISTINCT test.b):Int64]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ ",
+ )
+ }
}
diff --git a/datafusion/sqllogictest/test_files/single_distinct_to_groupby.slt b/datafusion/sqllogictest/test_files/single_distinct_to_groupby.slt
new file mode 100644
index 0000000000..afef4908e9
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/single_distinct_to_groupby.slt
@@ -0,0 +1,453 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Tests for the `single_distinct_aggregation_to_group_by` rule, which rewrites a
+# single `AGG(DISTINCT x)` into a two phase group by. Every result here is
+# asserted twice: once with the optimizer disabled
+# (`datafusion.optimizer.max_passes = 0`, so the aggregate runs as written) and
+# once with it enabled. The two must agree.
+#
+# A non-distinct `count` may only join the distinct aggregate when that
+# aggregate reports that it has no specialized `GroupsAccumulator` for its
+# argument types. `x` is the `VARCHAR` that qualifies, `xi` the `INT` that does
+# not; both hold the same values so the two sides can be compared directly.
+# An aggregate that reports nothing, which is every aggregate but `count`, also
+# does not qualify.
+
+statement ok
+CREATE TABLE t(g INT, x VARCHAR, xi INT, v INT);
+
+# g = 1: four rows, two distinct x, one NULL x, one NULL v
+# g = 2: two rows, x is NULL throughout
+# g = 3: a single row
+statement ok
+INSERT INTO t VALUES
+ (1, '10', 10, 100),
+ (1, '10', 10, NULL),
+ (1, NULL, NULL, 5),
+ (1, '20', 20, 7),
+ (2, NULL, NULL, NULL),
+ (2, NULL, NULL, 3),
+ (3, '30', 30, 1);
+
+statement ok
+CREATE TABLE empty_t(g INT, x VARCHAR, xi INT, v INT);
+
+statement ok
+set datafusion.explain.logical_plan_only = true;
+
+########
+# The rewrite fires for a non-distinct count next to the distinct aggregate
+########
+
+query TT
+EXPLAIN SELECT g, count(*) AS records, count(DISTINCT x) AS distinct_x FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END AS records, count(alias1) AS distinct_x
+02)--Aggregate: groupBy=[[t.g]], aggr=[[sum(alias2), count(alias1)]]
+03)----Aggregate: groupBy=[[t.g, t.x AS alias1]], aggr=[[count(Int64(1)) AS alias2]]
+04)------TableScan: t projection=[g, x]
+
+# A count next to min, max and sum, all beside the distinct count
+query TT
+EXPLAIN
+SELECT g, count(*) AS records, count(DISTINCT x) AS distinct_x,
+ min(v) AS min_v, max(v) AS max_v, sum(v) AS sum_v
+FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END AS records, count(alias1) AS distinct_x, min(alias3) AS min_v, max(alias4) AS max_v, sum(alias5) AS sum_v
+02)--Aggregate: groupBy=[[t.g]], aggr=[[sum(alias2), count(alias1), min(alias3), max(alias4), sum(alias5)]]
+03)----Aggregate: groupBy=[[t.g, t.x AS alias1]], aggr=[[count(Int64(1)) AS alias2, min(t.v) AS alias3, max(t.v) AS alias4, sum(CAST(t.v AS Int64)) AS alias5]]
+04)------TableScan: t projection=[g, x, v]
+
+# A count with a FILTER still blocks the rewrite: the filter is per input row,
+# and the inner aggregate has already collapsed those rows
+query TT
+EXPLAIN SELECT g, count(*) FILTER (WHERE v > 1) AS records, count(DISTINCT x) AS distinct_x FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, count(Int64(1)) FILTER (WHERE t.v > Int64(1)) AS count(*) FILTER (WHERE t.v > Int64(1)) AS records, count(DISTINCT t.x) AS distinct_x
+02)--Aggregate: groupBy=[[t.g]], aggr=[[count(Int64(1)) FILTER (WHERE t.v > Int32(1)) AS count(Int64(1)) FILTER (WHERE t.v > Int64(1)), count(DISTINCT t.x)]]
+03)----TableScan: t projection=[g, x, v]
+
+# An unsupported non-distinct aggregate still blocks the rewrite
+query TT
+EXPLAIN SELECT g, avg(v) AS avg_v, count(DISTINCT x) AS distinct_x FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, avg(t.v) AS avg_v, count(DISTINCT t.x) AS distinct_x
+02)--Aggregate: groupBy=[[t.g]], aggr=[[avg(CAST(t.v AS Float64)), count(DISTINCT t.x)]]
+03)----TableScan: t projection=[g, x, v]
+
+########
+# The other side of the gate: a distinct argument that has its own
+# `GroupsAccumulator` keeps the count out
+########
+
+# `count(DISTINCT xi)` over an INT is served by
+# `PrimitiveDistinctCountGroupsAccumulator`, so the rewrite would add an inner
+# group by without taking anything off `GroupsAccumulatorAdapter`
+query TT
+EXPLAIN SELECT g, count(*) AS records, count(DISTINCT xi) AS distinct_xi FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, count(Int64(1)) AS count(*) AS records, count(DISTINCT t.xi) AS distinct_xi
+02)--Aggregate: groupBy=[[t.g]], aggr=[[count(Int64(1)), count(DISTINCT t.xi)]]
+03)----TableScan: t projection=[g, xi]
+
+# `sum` reports nothing about `GroupsAccumulator` support from the argument
+# types, and neither does any aggregate but `count`. Silence is not evidence
+# that the rewrite pays, so the count keeps this plan unrewritten as well.
+# Rewritten, this shape measured 3.15x the peak memory over 4,000,000 rows in
+# 2,000 groups
+query TT
+EXPLAIN SELECT g, count(*) AS records, sum(DISTINCT v) AS distinct_v FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, count(Int64(1)) AS count(*) AS records, sum(DISTINCT t.v) AS distinct_v
+02)--Aggregate: groupBy=[[t.g]], aggr=[[count(Int64(1)), sum(DISTINCT CAST(t.v AS Int64))]]
+03)----TableScan: t projection=[g, v]
+
+# `min(DISTINCT v)` is the same value as `min(v)`, so the unrewritten plan keeps
+# one scalar per group while the rewrite would build a row per distinct pair
+query TT
+EXPLAIN SELECT g, count(*) AS records, min(DISTINCT v) AS distinct_min_v FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, count(Int64(1)) AS count(*) AS records, min(DISTINCT t.v) AS distinct_min_v
+02)--Aggregate: groupBy=[[t.g]], aggr=[[count(Int64(1)), min(DISTINCT t.v)]]
+03)----TableScan: t projection=[g, v]
+
+# The gate covers only the count. A plan that already qualified through sum,
+# min or max is still rewritten over the same INT column
+query TT
+EXPLAIN SELECT g, sum(v) AS sum_v, count(DISTINCT xi) AS distinct_xi FROM t GROUP BY g;
+----
+logical_plan
+01)Projection: t.g, sum(alias2) AS sum_v, count(alias1) AS distinct_xi
+02)--Aggregate: groupBy=[[t.g]], aggr=[[sum(alias2), count(alias1)]]
+03)----Aggregate: groupBy=[[t.g, t.xi AS alias1]], aggr=[[sum(CAST(t.v AS Int64)) AS alias2]]
+04)------TableScan: t projection=[g, xi, v]
+
+statement ok
+set datafusion.explain.logical_plan_only = false;
+
+########
+# count(*) vs count(col) vs count(1), grouped
+########
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+query IIIIII
+SELECT g, count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(v) AS cnt_v,
+ count(DISTINCT x) AS distinct_x
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 4 3 3 2
+2 2 2 0 1 0
+3 1 1 1 1 1
+
+statement ok
+set datafusion.optimizer.max_passes = 3;
+
+query IIIIII
+SELECT g, count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(v) AS cnt_v,
+ count(DISTINCT x) AS distinct_x
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 4 3 3 2
+2 2 2 0 1 0
+3 1 1 1 1 1
+
+########
+# count(*) vs count(col) vs count(1), no GROUP BY
+########
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+query IIIII
+SELECT count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(v) AS cnt_v,
+ count(DISTINCT x) AS distinct_x
+FROM t;
+----
+7 7 4 5 3
+
+statement ok
+set datafusion.optimizer.max_passes = 3;
+
+query IIIII
+SELECT count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(v) AS cnt_v,
+ count(DISTINCT x) AS distinct_x
+FROM t;
+----
+7 7 4 5 3
+
+########
+# The distinct column is NULL for every row of the group
+########
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+query IIIII
+SELECT count(*) AS star, count(x) AS cnt_x, count(v) AS cnt_v, count(DISTINCT x) AS distinct_x,
+ max(v) AS max_v
+FROM t WHERE g = 2;
+----
+2 0 1 0 3
+
+statement ok
+set datafusion.optimizer.max_passes = 3;
+
+query IIIII
+SELECT count(*) AS star, count(x) AS cnt_x, count(v) AS cnt_v, count(DISTINCT x) AS distinct_x,
+ max(v) AS max_v
+FROM t WHERE g = 2;
+----
+2 0 1 0 3
+
+########
+# Empty input. Without a GROUP BY the aggregate still emits one row, and the
+# counts on it must be 0 rather than the NULL an unguarded `sum` would give.
+########
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+query IIIII
+SELECT count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(DISTINCT x) AS distinct_x,
+ max(v) AS max_v
+FROM empty_t;
+----
+0 0 0 0 NULL
+
+query IIIII
+SELECT count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(DISTINCT x) AS distinct_x,
+ max(v) AS max_v
+FROM t WHERE g = 99;
+----
+0 0 0 0 NULL
+
+query IIII
+SELECT g, count(*) AS star, count(x) AS cnt_x, count(DISTINCT x) AS distinct_x
+FROM empty_t GROUP BY g ORDER BY g;
+----
+
+statement ok
+set datafusion.optimizer.max_passes = 3;
+
+query IIIII
+SELECT count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(DISTINCT x) AS distinct_x,
+ max(v) AS max_v
+FROM empty_t;
+----
+0 0 0 0 NULL
+
+query IIIII
+SELECT count(*) AS star, count(1) AS one, count(x) AS cnt_x, count(DISTINCT x) AS distinct_x,
+ max(v) AS max_v
+FROM t WHERE g = 99;
+----
+0 0 0 0 NULL
+
+query IIII
+SELECT g, count(*) AS star, count(x) AS cnt_x, count(DISTINCT x) AS distinct_x
+FROM empty_t GROUP BY g ORDER BY g;
+----
+
+########
+# HAVING and ORDER BY read the rewritten count by name from the node above
+########
+
+statement ok
+set datafusion.explain.logical_plan_only = true;
+
+query TT
+EXPLAIN
+SELECT g, count(*) AS records, count(DISTINCT x) AS distinct_x
+FROM t GROUP BY g HAVING count(*) > 1 AND count(DISTINCT x) > 0
+ORDER BY count(*) DESC, g;
+----
+logical_plan
+01)Sort: records DESC NULLS FIRST, t.g ASC NULLS LAST
+02)--Projection: t.g, CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END AS records, count(alias1) AS distinct_x
+03)----Filter: CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END > Int64(1) AND count(alias1) > Int64(0)
+04)------Aggregate: groupBy=[[t.g]], aggr=[[sum(alias2), count(alias1)]]
+05)--------Aggregate: groupBy=[[t.g, t.x AS alias1]], aggr=[[count(Int64(1)) AS alias2]]
+06)----------TableScan: t projection=[g, x]
+
+statement ok
+set datafusion.explain.logical_plan_only = false;
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+query III
+SELECT g, count(*) AS records, count(DISTINCT x) AS distinct_x
+FROM t GROUP BY g HAVING count(*) > 1 AND count(DISTINCT x) > 0
+ORDER BY count(*) DESC, g;
+----
+1 4 2
+
+statement ok
+set datafusion.optimizer.max_passes = 3;
+
+query III
+SELECT g, count(*) AS records, count(DISTINCT x) AS distinct_x
+FROM t GROUP BY g HAVING count(*) > 1 AND count(DISTINCT x) > 0
+ORDER BY count(*) DESC, g;
+----
+1 4 2
+
+########
+# The same set of aggregates, over a join rather than a bare scan
+########
+
+statement ok
+CREATE TABLE join_t(x VARCHAR);
+
+statement ok
+INSERT INTO join_t VALUES ('10'), ('30');
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+query IIIIII
+SELECT t.g, count(*) AS records, count(DISTINCT t.x) AS distinct_x,
+ min(t.v) AS min_v, max(t.v) AS max_v, sum(t.v) AS sum_v
+FROM t JOIN join_t f ON t.x = f.x
+GROUP BY t.g ORDER BY t.g;
+----
+1 2 1 100 100 100
+3 1 1 1 1 1
+
+statement ok
+set datafusion.optimizer.max_passes = 3;
+
+query IIIIII
+SELECT t.g, count(*) AS records, count(DISTINCT t.x) AS distinct_x,
+ min(t.v) AS min_v, max(t.v) AS max_v, sum(t.v) AS sum_v
+FROM t JOIN join_t f ON t.x = f.x
+GROUP BY t.g ORDER BY t.g;
+----
+1 2 1 100 100 100
+3 1 1 1 1 1
+
+########
+# Both sides of the gate produce the same results as the unoptimized plan
+########
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+# gated: the count keeps the INT distinct unrewritten
+query IIIII
+SELECT g, count(*) AS records, count(DISTINCT xi) AS distinct_xi, sum(v) AS sum_v, max(v) AS max_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 2 112 100
+2 2 0 3 3
+3 1 1 1 1
+
+# rewritten: the same shape over the VARCHAR distinct
+query IIIII
+SELECT g, count(*) AS records, count(DISTINCT x) AS distinct_x, sum(v) AS sum_v, max(v) AS max_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 2 112 100
+2 2 0 3 3
+3 1 1 1 1
+
+# rewritten through sum alone, over the INT distinct the gate excludes
+query III
+SELECT g, count(DISTINCT xi) AS distinct_xi, sum(v) AS sum_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 2 112
+2 0 3
+3 1 1
+
+query II
+SELECT count(*) AS records, count(DISTINCT xi) AS distinct_xi FROM empty_t;
+----
+0 0
+
+# gated: the count keeps a distinct aggregate that reports nothing unrewritten
+query III
+SELECT g, count(*) AS records, sum(DISTINCT v) AS distinct_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 112
+2 2 3
+3 1 1
+
+statement ok
+set datafusion.optimizer.max_passes = 3;
+
+# gated: the count keeps the INT distinct unrewritten
+query IIIII
+SELECT g, count(*) AS records, count(DISTINCT xi) AS distinct_xi, sum(v) AS sum_v, max(v) AS max_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 2 112 100
+2 2 0 3 3
+3 1 1 1 1
+
+# rewritten: the same shape over the VARCHAR distinct
+query IIIII
+SELECT g, count(*) AS records, count(DISTINCT x) AS distinct_x, sum(v) AS sum_v, max(v) AS max_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 2 112 100
+2 2 0 3 3
+3 1 1 1 1
+
+# rewritten through sum alone, over the INT distinct the gate excludes
+query III
+SELECT g, count(DISTINCT xi) AS distinct_xi, sum(v) AS sum_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 2 112
+2 0 3
+3 1 1
+
+query II
+SELECT count(*) AS records, count(DISTINCT xi) AS distinct_xi FROM empty_t;
+----
+0 0
+
+# gated: the count keeps a distinct aggregate that reports nothing unrewritten
+query III
+SELECT g, count(*) AS records, sum(DISTINCT v) AS distinct_v
+FROM t GROUP BY g ORDER BY g;
+----
+1 4 112
+2 2 3
+3 1 1
+
+statement ok
+DROP TABLE join_t;
+
+statement ok
+DROP TABLE empty_t;
+
+statement ok
+DROP TABLE t;
diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
index 3716b0feba..813d0ed6c3 100644
--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
@@ -333,8 +333,67 @@ async fn simple_aggregate() -> Result<()> {
#[tokio::test]
async fn aggregate_distinct_with_having() -> Result<()> {
- roundtrip("SELECT a, count(distinct b) FROM data GROUP BY a, c HAVING count(b) > 100")
- .await
+ let ctx = create_context_without_single_distinct_to_group_by().await?;
+ roundtrip_with_ctx(
+ "SELECT a, count(distinct b) FROM data GROUP BY a, c HAVING count(b) > 100",
+ ctx,
+ )
+ .await?;
+ Ok(())
+}
+
+/// The query in [`aggregate_distinct_with_having`], through the default
+/// optimizer, where `single_distinct_aggregation_to_group_by` rewrites it into
+/// a two phase group by.
+///
+/// A rewritten plan does not round trip to an identical plan, so the identity
+/// test above turns the rule off. This one leaves it on and asserts what
+/// Substrait does carry: the output schema and the rows. The `HAVING` threshold
+/// is lowered from the identity test's so that there are rows to compare.
+#[tokio::test]
+async fn aggregate_distinct_with_having_rewritten() -> Result<()> {
+ let ctx = create_context().await?;
+ let plan = ctx
+ .sql("SELECT a, count(distinct b) FROM data GROUP BY a, c HAVING count(b) > 0")
+ .await?
+ .into_optimized_plan()?;
+
+ // The rule applied: the distinct argument is a grouping key of an inner
+ // aggregate, and the non-distinct `count` became a `sum` of its partial
+ // counts. Without this the test would keep passing over an unrewritten
+ // plan, which the identity test already covers.
+ assert_snapshot!(
+ plan,
+ @r"
+ Projection: data.a, count(alias1) AS count(DISTINCT data.b)
+ Filter: CASE WHEN sum(alias2) IS NOT NULL THEN sum(alias2) ELSE Int64(0) END > Int64(0)
+ Projection: data.a, count(alias1), sum(alias2)
+ Aggregate: groupBy=[[data.a, data.c]], aggr=[[count(alias1), sum(alias2)]]
+ Aggregate: groupBy=[[data.a, data.c, data.b AS alias1]], aggr=[[count(data.b) AS alias2]]
+ TableScan: data projection=[a, b, c]
+ "
+ );
+
+ let proto = to_substrait_plan(&plan, &ctx.state())?;
+ let plan2 = from_substrait_plan(&ctx.state(), &proto).await?;
+ let plan2 = ctx.state().optimize(&plan2)?;
+
+ assert_eq!(plan.schema(), plan2.schema());
+
+ let results = DataFrame::new(ctx.state(), plan2).collect().await?;
+ datafusion::assert_batches_sorted_eq!(
+ [
+ "+---+------------------------+",
+ "| a | count(DISTINCT data.b) |",
+ "+---+------------------------+",
+ "| 1 | 1 |",
+ "| 3 | 1 |",
+ "+---+------------------------+",
+ ],
+ &results
+ );
+
+ Ok(())
}
#[tokio::test]
@@ -2935,6 +2994,34 @@ async fn create_context() -> Result<SessionContext> {
create_context_with_dialect(None).await
}
+/// [`create_context`] with `single_distinct_aggregation_to_group_by` removed from
+/// the optimizer.
+///
+/// That rule rewrites a single `AGG(DISTINCT x)` into a two phase group by whose
+/// inner aggregate aliases its grouping and measure expressions (`alias1`,
+/// `alias2`). Substrait carries no names for those expressions, so the consumer
+/// derives them from the expressions themselves and a rewritten plan does not
+/// round trip to an identical plan. That holds for every output of the rule, not
+/// only for the query under test.
+///
+/// What a rewritten plan does carry through Substrait is covered by
+/// [`aggregate_distinct_with_having_rewritten`], on the default context.
+async fn create_context_without_single_distinct_to_group_by() -> Result<SessionContext> {
+ let ctx = create_context().await?;
+ let rules = ctx
+ .state()
+ .optimizer()
+ .rules
+ .iter()
+ .filter(|rule| rule.name() != "single_distinct_aggregation_to_group_by")
+ .cloned()
+ .collect();
+ let state = SessionStateBuilder::new_from_existing(ctx.state())
+ .with_optimizer_rules(rules)
+ .build();
+ Ok(SessionContext::new_with_state(state))
+}
+
async fn create_context_with_dialect(dialect: Option<Dialect>) -> Result<SessionContext> {
let mut session_config = SessionConfig::default();