Apache DataFusion SQL Query Engine
Fix correlated subquery empty defaults for regr_count and approx_distinct (#22319)
Nathan Bezualemcab69a1d4aa8
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 6512d9b432..0ced83f7b9 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -1204,7 +1204,7 @@ async fn window_using_aggregates() -> Result<()> {+-------------+----------+-----------------+---------------+--------+-----+------+----+------+| first_value | last_val | approx_distinct | approx_median | median | max | min | c2 | c3 |+-------------+----------+-----------------+---------------+--------+-----+------+----+------+- | | | | | | | | 1 | -85 |+ | | | 0 | | | | | 1 | -85 || -85 | -101 | 14 | -12.0 | -12.0 | 83 | -101 | 4 | -54 || -85 | -101 | 17 | -25.0 | -25.0 | 83 | -101 | 5 | -31 || -85 | -12 | 10 | -32.75 | -34.0 | 83 | -85 | 3 | 13 |diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex cc42b6c22b..306ec074d4 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -381,6 +381,14 @@ impl AggregateUDFImpl for ApproxDistinct {Ok(DataType::UInt64)}+ fn default_value(&self, _data_type: &DataType) -> Result<ScalarValue> {+ Ok(ScalarValue::UInt64(Some(0)))+ }++ fn is_nullable(&self) -> bool {+ false+ }+fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {let data_type = args.input_fields[0].data_type();match data_type {diff --git a/datafusion/functions-aggregate/src/regr.rs b/datafusion/functions-aggregate/src/regr.rsindex 3a68672abb..3d5bbf1eda 100644--- a/datafusion/functions-aggregate/src/regr.rs+++ b/datafusion/functions-aggregate/src/regr.rs@@ -457,6 +457,18 @@ impl AggregateUDFImpl for Regr {}}+ fn default_value(&self, _data_type: &DataType) -> Result<ScalarValue> {+ if self.regr_type == RegrType::Count {+ Ok(ScalarValue::UInt64(Some(0)))+ } else {+ Ok(ScalarValue::Float64(None))+ }+ }++ fn is_nullable(&self) -> bool {+ self.regr_type != RegrType::Count+ }+fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {Ok(Box::new(RegrAccumulator::try_new(&self.regr_type)?))}diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex 2a71205c64..9490af0e59 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -35,8 +35,8 @@ use datafusion_expr::utils::{collect_subquery_cols, conjunction, find_join_exprs, split_conjunction,};use datafusion_expr::{- BinaryExpr, Cast, EmptyRelation, Expr, FetchType, LogicalPlan, LogicalPlanBuilder,- Operator, expr, lit,+ BinaryExpr, Cast, EmptyRelation, Expr, ExprSchemable, FetchType, LogicalPlan,+ LogicalPlanBuilder, Operator, expr, lit,};/// This struct rewrite the sub query plan by pull up the correlated@@ -512,18 +512,12 @@ fn agg_exprs_evaluation_result_on_empty_batch(let result_expr = e.clone().transform_up(|expr| {- let new_expr = match expr {- Expr::AggregateFunction(expr::AggregateFunction { func, .. }) => {- if func.name() == "count" {- Transformed::yes(Expr::Literal(- ScalarValue::Int64(Some(0)),- None,- ))- } else {- Transformed::yes(Expr::Literal(ScalarValue::Null, None))- }- }- _ => Transformed::no(expr),+ let new_expr = if let Expr::AggregateFunction(agg) = &expr {+ let return_type = expr.get_type(schema.as_ref())?;+ let default_value = agg.func.default_value(&return_type)?;+ Transformed::yes(Expr::Literal(default_value, None))+ } else {+ Transformed::no(expr)};Ok(new_expr)})diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rsindex fee430047a..27da19024c 100644--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs@@ -819,7 +819,7 @@ mod tests {assert_optimized_plan_equal!(plan,@r#"- Projection: customer.c_custkey, CASE WHEN __scalar_sq_1.__always_true IS NULL THEN CASE WHEN CAST(NULL AS Boolean) THEN Utf8("a") ELSE Utf8("b") END ELSE __scalar_sq_1.CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END END AS CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END [c_custkey:Int64, CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END:Utf8;N]+ Projection: customer.c_custkey, CASE WHEN __scalar_sq_1.__always_true IS NULL THEN CASE WHEN CAST(Float64(NULL) AS Boolean) THEN Utf8("a") ELSE Utf8("b") END ELSE __scalar_sq_1.CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END END AS CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END [c_custkey:Int64, CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END:Utf8;N]Left Join: Filter: customer.c_custkey = __scalar_sq_1.o_custkey [c_custkey:Int64, c_name:Utf8, CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END:Utf8;N, o_custkey:Int64;N, __always_true:Boolean;N]TableScan: customer [c_custkey:Int64, c_name:Utf8]SubqueryAlias: __scalar_sq_1 [CASE WHEN max(orders.o_totalprice) THEN Utf8("a") ELSE Utf8("b") END:Utf8, o_custkey:Int64, __always_true:Boolean]diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 25f124f217..dd195b0ff4 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -888,6 +888,68 @@ SELECT t1_id, (SELECT count(*) FROM t2 WHERE t2.t2_int = t1.t1_int) as cnt from33 344 0+#correlated_scalar_subquery_non_count_agg_empty_defaults+query III rowsort+SELECT+ t1_id,+ (+ SELECT regr_count(1.0, 1.0)+ FROM t2+ WHERE t2.t2_int = t1.t1_int+ ) AS r,+ (+ SELECT approx_distinct(t2.t2_id)+ FROM t2+ WHERE t2.t2_int = t1.t1_int+ ) AS d+FROM t1+----+11 1 1+22 0 0+33 3 3+44 0 0++query II rowsort+SELECT+ t1_id,+ (+ SELECT regr_count(1.0, 1.0) + approx_distinct(t2.t2_id)+ FROM t2+ WHERE t2.t2_int = t1.t1_int+ ) AS combined+FROM t1+----+11 2+22 0+33 6+44 0++query I rowsort+SELECT t1_id+FROM t1+WHERE+ (+ SELECT approx_distinct(t2.t2_id)+ FROM t2+ WHERE t2.t2_int = t1.t1_int+ ) = 0+----+22+44++query I rowsort+SELECT t1_id+FROM t1+WHERE+ (+ SELECT regr_count(1.0, 1.0)+ FROM t2+ WHERE t2.t2_int = t1.t1_int+ ) = 0+----+22+44+#correlated_scalar_subquery_count_agg_with_aliasquery TTexplain SELECT t1_id, (SELECT count(*) as _cnt FROM t2 WHERE t2.t2_int = t1.t1_int) as cnt from t1