Skip to content

Apache DataFusion SQL Query Engine

Move filter for `HAVING` and `QUALIFY` when volatile aliases are used above projection (#26850)

Matthew Turner92290cb10fbe
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index 726af00d7a..3d6e729937 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -42,7 +42,9 @@ use datafusion_expr::ExpressionPlacement;
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_expr::equivalence::ProjectionMapping;
use datafusion_physical_expr::projection::Projector;
-use datafusion_physical_expr_common::physical_expr::{PhysicalExprRef, fmt_sql};
+use datafusion_physical_expr_common::physical_expr::{
+ PhysicalExprRef, fmt_sql, is_volatile,
+};
use datafusion_physical_expr_common::sort_expr::{
LexOrdering, LexRequirement, PhysicalSortExpr,
};
@@ -524,10 +526,17 @@ impl ExecutionPlan for ProjectionExec {
for filter in parent_filters {
// Check that every column is a valid output column of this
// projection, then replace each one with the expression at that
- // output position.
+ // output position. A filter that becomes volatile after the
+ // rewrite must stay above the projection: inlining a volatile
+ // expression evaluates it independently, so the filter would test
+ // a different value than the one the projection outputs.
if let Some(reassigned) = remapper.try_remap(&filter)? {
let rewritten = self.projection_expr().unproject_expr(&reassigned)?;
- child_parent_filters.push(PushedDownPredicate::supported(rewritten));
+ if is_volatile(&rewritten) {
+ child_parent_filters.push(PushedDownPredicate::unsupported(filter));
+ } else {
+ child_parent_filters.push(PushedDownPredicate::supported(rewritten));
+ }
} else {
child_parent_filters.push(PushedDownPredicate::unsupported(filter));
}
@@ -1574,6 +1583,7 @@ mod tests {
use datafusion_expr::{Operator, ScalarUDF};
use datafusion_functions::core::arrow_metadata::ArrowMetadataFunc;
+ use datafusion_functions::math::random::RandomFunc;
use datafusion_physical_expr::ScalarFunctionExpr;
use datafusion_physical_expr::expressions::{
BinaryExpr, CastExpr, Column, DynamicFilterPhysicalExpr, Literal, binary, col,
@@ -2236,6 +2246,82 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_filter_pushdown_volatile_projection_expr() -> Result<()> {
+ let input_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let input = Arc::new(StatisticsExec::new(
+ Statistics::new_unknown(&input_schema),
+ input_schema.clone(),
+ ));
+
+ // project "a", random() as "r"
+ let random_expr = Arc::new(ScalarFunctionExpr::new(
+ "random",
+ Arc::new(ScalarUDF::new_from_impl(RandomFunc::new())),
+ vec![],
+ Arc::new(Field::new("r", DataType::Float64, false)),
+ Arc::new(ConfigOptions::default()),
+ )) as Arc<dyn PhysicalExpr>;
+ let projection = ProjectionExec::try_new(
+ vec![
+ ProjectionExpr {
+ expr: Arc::new(Column::new("a", 0)),
+ alias: "a".to_string(),
+ },
+ ProjectionExpr {
+ expr: random_expr,
+ alias: "r".to_string(),
+ },
+ ],
+ input,
+ )?;
+
+ // filter "a > 5" is deterministic and is pushed down, rewritten to
+ // reference the input column
+ let deterministic_filter = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Gt,
+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),
+ )) as Arc<dyn PhysicalExpr>;
+
+ // filter "r > 0.5" must stay above the projection: unprojecting it
+ // would inline an independent evaluation of random(), so the filter
+ // would test a different value than the one the projection outputs
+ let volatile_filter = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("r", 1)),
+ Operator::Gt,
+ Arc::new(Literal::new(ScalarValue::Float64(Some(0.5)))),
+ )) as Arc<dyn PhysicalExpr>;
+
+ let description = projection.gather_filters_for_pushdown(
+ FilterPushdownPhase::Post,
+ vec![deterministic_filter, Arc::clone(&volatile_filter)],
+ &ConfigOptions::default(),
+ )?;
+
+ let expected_pushed = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Gt,
+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),
+ )) as Arc<dyn PhysicalExpr>;
+
+ let pushed_filters = &description.parent_filters()[0];
+ assert_eq!(pushed_filters.len(), 2);
+ assert!(matches!(pushed_filters[0].discriminant, PushedDown::Yes));
+ assert_eq!(
+ format!("{}", pushed_filters[0].predicate),
+ format!("{}", expected_pushed)
+ );
+ assert!(matches!(pushed_filters[1].discriminant, PushedDown::No));
+ // the unsupported filter is kept unchanged
+ assert_eq!(
+ format!("{}", pushed_filters[1].predicate),
+ format!("{}", volatile_filter)
+ );
+
+ Ok(())
+ }
+
#[test]
fn test_filter_pushdown_with_multiple_aliases() -> Result<()> {
let input_schema = Schema::new(vec![
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs
index b9812f8386..59cef5df0e 100644
--- a/datafusion/sql/src/select.rs
+++ b/datafusion/sql/src/select.rs
@@ -12,9 +12,10 @@ use crate::query::to_order_by_exprs_with_select;
use crate::utils::{
CheckColumnsMustReferenceAggregatePurpose, CheckColumnsSatisfyExprsPurpose,
DedupedProjection, check_columns_satisfy_exprs, extract_aliases, rebase_expr,
- resolve_aliases_to_exprs, resolve_columns, resolve_positions_to_exprs,
- rewrite_recursive_unnest_bottom_up, rewrite_recursive_unnests_bottom_up,
- substitute_top_level_alias, substitute_top_level_aliases_in_sorts,
+ references_volatile_alias, resolve_aliases_to_exprs, resolve_columns,
+ resolve_positions_to_exprs, rewrite_recursive_unnest_bottom_up,
+ rewrite_recursive_unnests_bottom_up, substitute_top_level_alias,
+ substitute_top_level_aliases_in_sorts,
};
use arrow::datatypes::DataType;
@@ -34,7 +35,8 @@ use datafusion_expr::expr_rewriter::{
};
use datafusion_expr::select_expr::SelectExpr;
use datafusion_expr::utils::{
- expr_as_column_expr, expr_to_columns, find_aggregate_exprs, find_window_exprs,
+ conjunction, expr_as_column_expr, expr_to_columns, find_aggregate_exprs,
+ find_window_exprs, split_conjunction_owned,
};
use datafusion_expr::{
Aggregate, Expr, Filter, GroupingSet, LogicalPlan, LogicalPlanBuilder,
@@ -191,10 +193,14 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
})
.collect::<Result<Vec<Expr>>>()?;
- // Optionally the HAVING expression.
+ // Optionally the HAVING expression. Conjuncts that reference a
+ // volatile alias are deferred to `having_post_projection` and planned
+ // above the final projection (see [`references_volatile_alias`]); the
+ // remaining conjuncts take the normal path below the projection.
+ let mut having_post_projection: Option<Expr> = None;
let having_expr_opt = select
.having
- .map::<Result<Expr>, _>(|having_expr| {
+ .map::<Result<Option<Expr>>, _>(|having_expr| {
self.warn_on_null_equality_predicate(&having_expr);
let window_span = self.window_function_span(&having_expr, &alias_map);
let having_expr = self.sql_expr_to_logical_expr(
@@ -202,6 +208,34 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
&combined_schema,
planner_context,
)?;
+ // HAVING is evaluated before window functions are computed, so
+ // they may not appear there (checked on a fully dereferenced
+ // copy so that an alias of a window function is rejected too)
+ reject_window_functions(
+ &resolve_aliases_to_exprs(having_expr.clone(), &alias_map)?,
+ "HAVING",
+ QUALIFY_HELP,
+ window_span,
+ )?;
+ let mut deferred_conjuncts = vec![];
+ let mut normal_conjuncts = vec![];
+ for conjunct in split_conjunction_owned(having_expr) {
+ if references_volatile_alias(&conjunct, &alias_map)? {
+ deferred_conjuncts.push(conjunct);
+ } else {
+ normal_conjuncts.push(conjunct);
+ }
+ }
+ // Volatile aliases are never dereferenced; see
+ // [`references_volatile_alias`] for why.
+ having_post_projection = match conjunction(deferred_conjuncts) {
+ Some(expr) => {
+ let expr = normalize_col(expr, &projected_plan)?;
+ let (expr, _) = expr.infer_placeholder_types(&combined_schema)?;
+ Some(expr)
+ }
+ None => None,
+ };
// This step "dereferences" any aliases in the HAVING clause.
//
// This is how we support queries with HAVING expressions that
@@ -215,22 +249,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
//
// SELECT c1, MAX(c2) AS m FROM t GROUP BY c1 HAVING MAX(c2) > 10;
//
- let having_expr = resolve_aliases_to_exprs(having_expr, &alias_map)?;
- // HAVING is evaluated before window functions are computed, so
- // they may not appear there (checked after alias resolution so
- // that an alias of a window function is rejected too)
- reject_window_functions(
- &having_expr,
- "HAVING",
- QUALIFY_HELP,
- window_span,
- )?;
- let having_expr = normalize_col(having_expr, &projected_plan)?;
- let (having_expr, _) =
- having_expr.infer_placeholder_types(&combined_schema)?;
- Ok(having_expr)
+ match conjunction(normal_conjuncts) {
+ Some(expr) => {
+ let expr = resolve_aliases_to_exprs(expr, &alias_map)?;
+ let expr = normalize_col(expr, &projected_plan)?;
+ let (expr, _) = expr.infer_placeholder_types(&combined_schema)?;
+ Ok(Some(expr))
+ }
+ None => Ok(None),
+ }
})
- .transpose()?;
+ .transpose()?
+ .flatten();
// All of the group by expressions
let group_by_exprs = if let GroupByExpr::Expressions(exprs, _) = select.group_by {
@@ -284,15 +314,37 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
.collect()
};
- // Optionally the QUALIFY expression.
+ // Optionally the QUALIFY expression. Like HAVING, conjuncts that
+ // reference a volatile alias are deferred to
+ // `qualify_post_projection` and planned above the final projection.
+ let mut qualify_post_projection: Option<Expr> = None;
let qualify_expr_opt = select
.qualify
- .map::<Result<Expr>, _>(|qualify_expr| {
+ .map::<Result<Option<Expr>>, _>(|qualify_expr| {
let qualify_expr = self.sql_expr_to_logical_expr(
qualify_expr,
&combined_schema,
planner_context,
)?;
+ let mut deferred_conjuncts = vec![];
+ let mut normal_conjuncts = vec![];
+ for conjunct in split_conjunction_owned(qualify_expr) {
+ if references_volatile_alias(&conjunct, &alias_map)? {
+ deferred_conjuncts.push(conjunct);
+ } else {
+ normal_conjuncts.push(conjunct);
+ }
+ }
+ // Volatile aliases are never dereferenced; see
+ // [`references_volatile_alias`] for why.
+ qualify_post_projection = match conjunction(deferred_conjuncts) {
+ Some(expr) => {
+ let expr = normalize_col(expr, &projected_plan)?;
+ let (expr, _) = expr.infer_placeholder_types(&combined_schema)?;
+ Some(expr)
+ }
+ None => None,
+ };
// This step "dereferences" any aliases in the QUALIFY clause.
//
// This is how we support queries with QUALIFY expressions that
@@ -306,13 +358,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
//
// select row_number() over (PARTITION BY id) as rk from users qualify row_number() over (PARTITION BY id) > 1;
//
- let qualify_expr = resolve_aliases_to_exprs(qualify_expr, &alias_map)?;
- let qualify_expr = normalize_col(qualify_expr, &projected_plan)?;
- let (qualify_expr, _) =
- qualify_expr.infer_placeholder_types(&combined_schema)?;
- Ok(qualify_expr)
+ match conjunction(normal_conjuncts) {
+ Some(expr) => {
+ let expr = resolve_aliases_to_exprs(expr, &alias_map)?;
+ let expr = normalize_col(expr, &projected_plan)?;
+ let (expr, _) = expr.infer_placeholder_types(&combined_schema)?;
+ Ok(Some(expr))
+ }
+ None => Ok(None),
+ }
})
- .transpose()?;
+ .transpose()?
+ .flatten();
// The outer expressions we will search through for aggregates.
// First, find aggregates in SELECT, HAVING, and QUALIFY
@@ -358,7 +415,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
&aggr_exprs,
)?
} else {
- match having_expr_opt {
+ match having_expr_opt.as_ref().or(having_post_projection.as_ref()) {
Some(having_expr) => {
return plan_err!(
"HAVING clause references: {having_expr} must appear in the GROUP BY clause or be used in an aggregate function"
@@ -367,7 +424,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
None => AggregatePlanResult {
plan: base_plan.clone(),
select_exprs: select_exprs.clone(),
- having_expr: having_expr_opt,
+ having_expr: None,
qualify_expr: qualify_expr_opt,
order_by_exprs: order_by_rex,
on_exprs: on_exprs_pre_aggr,
@@ -464,6 +521,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
plan
};
+ // A QUALIFY that was deferred to a post-projection filter still
+ // requires window functions elsewhere in the query. A window function
+ // inside the deferred predicate itself is rejected later with a more
+ // specific error, so it is not counted here.
+ if let Some(qualify_expr) = &qualify_post_projection
+ && window_func_exprs.is_empty()
+ && find_window_exprs(std::iter::once(qualify_expr)).is_empty()
+ {
+ return plan_err!(
+ "QUALIFY clause requires window functions in the SELECT list or QUALIFY clause"
+ );
+ }
+
// Process distinct clause. For `DISTINCT ON` combined with
// aggregation, GROUP BY, or window functions we apply DistinctOn
// *before* the final projection so grouping columns and ORDER BY
@@ -472,12 +542,31 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// list is wrapped in FIRST_VALUE during lowering).
let plan = match select.distinct {
None | Some(Distinct::All) => {
- self.try_process_unnest(plan, select_exprs_post_aggr)?
+ let plan = self.try_process_unnest(plan, select_exprs_post_aggr)?;
+ let plan =
+ self.filter_post_projection(plan, having_post_projection, "HAVING")?;
+ self.filter_post_projection(plan, qualify_post_projection, "QUALIFY")?
}
Some(Distinct::Distinct) => {
let plan = self.try_process_unnest(plan, select_exprs_post_aggr)?;
+ // HAVING and QUALIFY are evaluated before DISTINCT
+ let plan =
+ self.filter_post_projection(plan, having_post_projection, "HAVING")?;
+ let plan = self.filter_post_projection(
+ plan,
+ qualify_post_projection,
+ "QUALIFY",
+ )?;
LogicalPlanBuilder::from(plan).distinct()?.build()?
}
+ Some(Distinct::On(_))
+ if having_post_projection.is_some()
+ || qualify_post_projection.is_some() =>
+ {
+ return plan_err!(
+ "HAVING and QUALIFY expressions that reference aliases of volatile expressions are not supported with DISTINCT ON. Use a subquery instead"
+ );
+ }
Some(Distinct::On(_)) => {
if aggr_exprs.is_empty()
&& group_by_exprs.is_empty()
@@ -557,6 +646,37 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
Ok(plan)
}
+ /// Plans a deferred HAVING / QUALIFY predicate — one that references a
+ /// volatile alias (see [`references_volatile_alias`]) — as a `Filter`
+ /// above the final projection, where the alias is a real column.
+ /// Everything the predicate references must be in the projection output;
+ /// anything else is rejected with a suggestion to use a subquery.
+ fn filter_post_projection(
+ &self,
+ plan: LogicalPlan,
+ filter_expr: Option<Expr>,
+ clause: &str,
+ ) -> Result<LogicalPlan> {
+ let Some(expr) = filter_expr else {
+ return Ok(plan);
+ };
+ if !find_aggregate_exprs(std::iter::once(&expr)).is_empty()
+ || !find_window_exprs(std::iter::once(&expr)).is_empty()
+ {
+ return plan_err!(
+ "{clause} clauses that reference an alias of a volatile expression cannot also contain aggregate or window functions. Add the function to the SELECT list and reference it by alias, or use a subquery"
+ );
+ }
+ for column in expr.column_refs() {
+ if plan.schema().qualified_field_from_column(column).is_err() {
+ return plan_err!(
+ "{clause} clauses that reference an alias of a volatile expression can only reference columns from the SELECT list, but {column} is not in the SELECT list. Use a subquery instead"
+ );
+ }
+ }
+ LogicalPlanBuilder::from(plan).filter(expr)?.build()
+ }
+
/// Try converting Expr(Unnest(Expr)) to Projection/Unnest/Projection
pub(super) fn try_process_unnest(
&self,
diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs
index 415db2b6ee..d581cf3447 100644
--- a/datafusion/sql/src/utils.rs
+++ b/datafusion/sql/src/utils.rs
@@ -259,6 +259,27 @@ pub(crate) fn resolve_positions_to_exprs(
}
}
+/// Returns true if `expr` references (as a bare unqualified column) any
+/// SELECT alias whose underlying expression is volatile.
+///
+/// Such references must not be dereferenced to the underlying expression:
+/// every copy of a volatile expression is evaluated independently, so the
+/// rewritten clause would test a different value than the one the query
+/// returns. See <https://github.com/apache/datafusion/issues/7976>.
+pub(crate) fn references_volatile_alias(
+ expr: &Expr,
+ aliases: &HashMap<String, Expr>,
+) -> Result<bool> {
+ expr.exists(|nested_expr| {
+ Ok(matches!(
+ nested_expr,
+ Expr::Column(c)
+ if c.relation.is_none()
+ && aliases.get(&c.name).is_some_and(Expr::is_volatile)
+ ))
+ })
+}
+
/// Rebuilds an `Expr` with columns that refer to aliases replaced by the
/// alias' underlying `Expr`.
pub(crate) fn resolve_aliases_to_exprs(
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index 58f37d593e..112e257677 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -9193,6 +9193,94 @@ select 0 from having_test having max(v1) = 4
query error
select * from having_test group by v1 having max(v1) = 3
+# HAVING that references the alias of a volatile expression must filter on
+# the value the query returns: every returned row must satisfy the predicate
+# on the returned value of r; see
+# https://github.com/apache/datafusion/issues/7976
+query B
+select count(*) > 0 from (
+ select value as v, random() as r
+ from generate_series(1, 1000)
+ group by v
+ having r > 0.5
+) where r <= 0.5;
+----
+false
+
+# the predicate is planned as a filter above the projection, where the alias
+# is a real column, instead of dereferencing the alias into a fresh random()
+query TT
+EXPLAIN select v1, random() as r from having_test group by v1 having r > 0.5
+----
+logical_plan
+01)Filter: r > Float64(0.5)
+02)--Projection: having_test.v1, random() AS r
+03)----Aggregate: groupBy=[[having_test.v1]], aggr=[[]]
+04)------TableScan: having_test projection=[v1]
+physical_plan
+01)FilterExec: r@1 > 0.5
+02)--ProjectionExec: expr=[v1@0 as v1, random() as r]
+03)----AggregateExec: mode=FinalPartitioned, gby=[v1@0 as v1], aggr=[]
+04)------RepartitionExec: partitioning=Hash([v1@0], 4), input_partitions=1
+05)--------AggregateExec: mode=Partial, gby=[v1@0 as v1], aggr=[]
+06)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# a non-volatile alias next to a volatile alias is also kept as a column
+# reference and resolved against the projection
+query I
+select count(*) from (select v1, max(v2) as m, random() as r from having_test group by v1 having r >= 0 and m > 0)
+----
+3
+
+# conjuncts are planned independently: an aggregate conjunct filters below
+# the projection as usual, a volatile-alias conjunct filters above it
+query TT
+EXPLAIN select v1, random() as r from having_test group by v1 having r > 0.5 and max(v2) > 0
+----
+logical_plan
+01)Filter: r > Float64(0.5)
+02)--Projection: having_test.v1, random() AS r
+03)----Filter: max(having_test.v2) > Int32(0)
+04)------Aggregate: groupBy=[[having_test.v1]], aggr=[[max(having_test.v2)]]
+05)--------TableScan: having_test projection=[v1, v2]
+physical_plan
+01)FilterExec: r@1 > 0.5
+02)--ProjectionExec: expr=[v1@0 as v1, random() as r]
+03)----FilterExec: max(having_test.v2)@1 > 0, projection=[v1@0]
+04)------AggregateExec: mode=FinalPartitioned, gby=[v1@0 as v1], aggr=[max(having_test.v2)]
+05)--------RepartitionExec: partitioning=Hash([v1@0], 4), input_partitions=1
+06)----------AggregateExec: mode=Partial, gby=[v1@0 as v1], aggr=[max(having_test.v2)]
+07)------------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# a volatile alias and an aggregate in the same conjunct cannot be split
+query error DataFusion error: Error during planning: HAVING clauses that reference an alias of a volatile expression cannot also contain aggregate or window functions\. Add the function to the SELECT list and reference it by alias, or use a subquery
+select v1, random() as r from having_test group by v1 having r > max(v2)
+
+# an OR predicate is planned as one filter. The volatile alias forces that
+# filter above the projection, but max(v2) is not available there
+query error DataFusion error: Error during planning: HAVING clauses that reference an alias of a volatile expression cannot also contain aggregate or window functions\. Add the function to the SELECT list and reference it by alias, or use a subquery
+select v1, random() as r from having_test group by v1 having r > 0.5 or max(v2) > 0
+
+# a group-by column conjunct splits off and plans below the projection even
+# when it is not in the SELECT list
+query I
+select count(*) from (select v1, random() as r from having_test group by v1, v2 having r >= 0 and v2 > 0)
+----
+3
+
+# a column that is not in the SELECT list cannot appear in the same conjunct
+# as a volatile alias reference
+query error DataFusion error: Error during planning: HAVING clauses that reference an alias of a volatile expression can only reference columns from the SELECT list, but having_test\.v2 is not in the SELECT list\. Use a subquery instead
+select v1, random() as r from having_test group by v1, v2 having r > 0.5 or v2 > 0
+
+# volatile alias in HAVING still requires aggregation
+query error DataFusion error: Error during planning: HAVING clause references: r > Float64\(0\.5\) must appear in the GROUP BY clause or be used in an aggregate function
+select random() as r from having_test having r > 0.5
+
+# volatile alias in HAVING is not supported with DISTINCT ON
+query error DataFusion error: Error during planning: HAVING and QUALIFY expressions that reference aliases of volatile expressions are not supported with DISTINCT ON\. Use a subquery instead
+select distinct on (v1) v1, random() as r from having_test group by v1 having r > 0.5
+
statement ok
drop table having_test
diff --git a/datafusion/sqllogictest/test_files/qualify.slt b/datafusion/sqllogictest/test_files/qualify.slt
index 43d9b6d3ca..47aafc9148 100644
--- a/datafusion/sqllogictest/test_files/qualify.slt
+++ b/datafusion/sqllogictest/test_files/qualify.slt
@@ -345,6 +345,71 @@ physical_plan
11)--------------------AggregateExec: mode=Partial, gby=[dept@1 as dept], aggr=[sum(users.salary)]
12)----------------------DataSourceExec: partitions=1, partition_sizes=[1]
+# QUALIFY that references the alias of a volatile expression must filter on
+# the value the query returns: every returned row must satisfy the predicate
+# on the returned value of r; see
+# https://github.com/apache/datafusion/issues/7976
+query B
+select count(*) > 0 from (
+ select row_number() over () as rn, random() as r
+ from generate_series(1, 1000)
+ qualify rn >= 1 and r > 0.5
+) where r <= 0.5;
+----
+false
+
+# the predicate is planned as a filter above the projection, where the alias
+# is a real column, instead of dereferencing the alias into a fresh random();
+# the deterministic rn >= 1 conjunct is still pushed below the projection
+query TT
+EXPLAIN SELECT id, ROW_NUMBER() OVER (ORDER BY id) AS rn, random() AS r
+FROM users
+QUALIFY rn >= 1 AND r > 0.5;
+----
+logical_plan
+01)Filter: r > Float64(0.5)
+02)--Projection: users.id, row_number() ORDER BY [users.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rn, random() AS r
+03)----Filter: row_number() ORDER BY [users.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW >= UInt64(1)
+04)------WindowAggr: windowExpr=[[row_number() ORDER BY [users.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
+05)--------TableScan: users projection=[id]
+physical_plan
+01)FilterExec: r@2 > 0.5
+02)--ProjectionExec: expr=[id@0 as id, row_number() ORDER BY [users.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rn, random() as r]
+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
+04)------FilterExec: row_number() ORDER BY [users.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 >= 1
+05)--------BoundedWindowAggExec: wdw=[row_number() ORDER BY [users.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [users.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+06)----------SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+07)------------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# conjuncts are planned independently: a raw window function conjunct takes
+# the normal QUALIFY path, a volatile-alias conjunct filters above the
+# projection on the value the query returns
+query B
+SELECT count(*) > 0 FROM (
+ SELECT random() AS r
+ FROM generate_series(1, 1000)
+ QUALIFY ROW_NUMBER() OVER () >= 1 AND r > 0.5
+) WHERE r <= 0.5;
+----
+false
+
+# a volatile alias and a window function in the same conjunct cannot be split
+query error DataFusion error: Error during planning: QUALIFY clauses that reference an alias of a volatile expression cannot also contain aggregate or window functions\. Add the function to the SELECT list and reference it by alias, or use a subquery
+SELECT id, random() AS r
+FROM users
+QUALIFY r > ROW_NUMBER() OVER (ORDER BY id);
+
+# an OR predicate is planned as one filter. The volatile alias forces that
+# filter above the projection, but the window function is not available there
+query error DataFusion error: Error during planning: QUALIFY clauses that reference an alias of a volatile expression cannot also contain aggregate or window functions\. Add the function to the SELECT list and reference it by alias, or use a subquery
+SELECT id, random() AS r
+FROM users
+QUALIFY ROW_NUMBER() OVER (ORDER BY id) >= 1 OR r > 0.5;
+
+# volatile alias in QUALIFY still requires a window function
+query error DataFusion error: Error during planning: QUALIFY clause requires window functions in the SELECT list or QUALIFY clause
+SELECT id, random() AS r FROM users QUALIFY r > 0.5;
+
# Clean up
statement ok
DROP TABLE users;