Apache DataFusion SQL Query Engine
fix: reject window functions in WHERE and HAVING during logical planning (#25139)
Adrian Garcia Badaracco7e5f40a69aae
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 882956a63c..f00a1b555a 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -1270,16 +1270,20 @@ impl DefaultPhysicalPlanner {LogicalPlan::SubqueryAlias(_) => children.one()?,LogicalPlan::Limit(limit) => {let input = children.one()?;+ // `get_skip_type` / `get_fetch_type` only return a non literal+ // type for an expression that is presentlet SkipType::Literal(skip) = limit.get_skip_type()? else {+ let skip = limit.skip.as_deref().map(ToString::to_string);return not_impl_err!(- "Unsupported OFFSET expression: {:?}",- limit.skip+ "Unsupported OFFSET expression: {}",+ skip.unwrap_or_default());};let FetchType::Literal(fetch) = limit.get_fetch_type()? else {+ let fetch = limit.fetch.as_deref().map(ToString::to_string);return not_impl_err!(- "Unsupported LIMIT expression: {:?}",- limit.fetch+ "Unsupported LIMIT expression: {}",+ fetch.unwrap_or_default());};diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 9b0ca1b276..392a6391bd 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -5334,6 +5334,25 @@ async fn consecutive_projection_same_schema() -> Result<()> {Ok(())}+#[tokio::test]+async fn window_function_in_group_by_is_rejected() -> Result<()> {+ // https://github.com/apache/datafusion/issues/4610+ //+ // The SQL planner rejects this placement itself, but the DataFrame API+ // has no such check, so the physical planner's backstop is what reports it+ let err = test_table()+ .await?+ .aggregate(vec![row_number()], vec![count(col("c1"))])?+ .collect()+ .await+ .expect_err("a window function cannot be a grouping expression");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function 'row_number() ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING' is not supported in this position. Window functions are supported in the SELECT list, ORDER BY, DISTINCT ON and QUALIFY"+ );+ Ok(())+}+async fn create_test_table(name: &str) -> Result<DataFrame> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex b2522c8533..36aa67bbe7 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -3131,6 +3131,50 @@ mod tests {Ok(())}+ #[test]+ fn plan_builder_filter_rejects_window_functions() -> Result<()> {+ // https://github.com/apache/datafusion/issues/4610+ let sum_over = |arg| {+ Expr::from(expr::WindowFunction::new(+ crate::WindowFunctionDefinition::AggregateUDF(+ crate::test::function_stub::sum_udaf(),+ ),+ vec![arg],+ ))+ };+ let scan = || table_scan(Some("employee_csv"), &employee_schema(), Some(vec![4]));++ let err = scan()?+ .filter(sum_over(col("salary")).gt(lit(10)))+ .expect_err("window functions in a filter should be rejected");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in filter predicates: 'sum(employee_csv.salary) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ let err = scan()?+ .having(sum_over(col("salary")).gt(lit(10)))+ .expect_err("window functions in a having should be rejected");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in filter predicates: 'sum(employee_csv.salary) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ // computing the window function first and filtering on its result is+ // the supported form+ let plan = scan()?+ .window(vec![sum_over(col("salary")).alias("total")])?+ .filter(col("total").gt(lit(10)))?+ .build()?;+ assert_snapshot!(plan, @r"+ Filter: total > Int32(10)+ WindowAggr: windowExpr=[[sum(employee_csv.salary) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS total]]+ TableScan: employee_csv projection=[salary]+ ");++ Ok(())+ }+#[test]fn test_join_metadata() -> Result<()> {let left_schema = DFSchema::new_with_metadata(diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex ab1ee1cdf4..cd0aee1322 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -41,9 +41,10 @@ use crate::logical_plan::display::{GraphvizVisitor, IndentVisitor};use crate::logical_plan::extension::UserDefinedLogicalNode;use crate::logical_plan::{DmlStatement, Statement, WriteOp};use crate::utils::{- check_aggregate_and_window_nesting, enumerate_grouping_sets, expr_to_columns,- exprlist_to_fields, find_out_reference_exprs, grouping_set_expr_count,- grouping_set_to_exprlist, merge_schema, split_conjunction,+ check_aggregate_and_window_nesting, check_no_window_functions,+ enumerate_grouping_sets, expr_to_columns, exprlist_to_fields,+ find_out_reference_exprs, grouping_set_expr_count, grouping_set_to_exprlist,+ merge_schema, split_conjunction,};use crate::{BinaryExpr, CreateMemoryTable, CreateView, Execute, Expr, ExprSchemable, GroupingSet,@@ -2822,12 +2823,13 @@ pub struct Filter {impl Filter {/// Create a new filter operator.///- /// Skips the type-checking and dealiasing done in [Self::try_new].- /// For internal use in DataFusion only.+ /// Skips the type-checking, window function check and dealiasing done in+ /// [Self::try_new]. For internal use in DataFusion only.////// **Preconditions:**/// - the `predicate` expression returns a boolean value/// - the `predicate` expression is not aliased+ /// - the `predicate` expression contains no window function calls#[doc(hidden)]pub fn new(predicate: Expr, input: Arc<LogicalPlan>) -> Self {Self { predicate, input }@@ -2835,6 +2837,10 @@ impl Filter {/// Create a new filter operator.///+ /// Returns an error if the predicate is not boolean or contains a window+ /// function call, which cannot be evaluated by a filter (see+ /// [`check_no_window_functions`]).+ ////// Notes: as Aliases have no effect on the output of a filter operator,/// they are removed from the predicate expression.pub fn try_new(predicate: Expr, input: Arc<LogicalPlan>) -> Result<Self> {@@ -2853,6 +2859,11 @@ impl Filter {}fn try_new_internal(predicate: Expr, input: Arc<LogicalPlan>) -> Result<Self> {+ // Filters are evaluated before window functions are computed, so a+ // window call in the predicate has no physical equivalent. Reject it+ // here rather than failing during physical planning.+ check_no_window_functions(&predicate, "filter predicates")?;+// Filter predicates must return a boolean value so we try and validate that here.// Note that it is not always possible to resolve the predicate expression during plan// construction (such as with correlated subqueries) so we make a best effort here anddiff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rsindex 6c1dbacf4b..4ca12c5e03 100644--- a/datafusion/expr/src/utils.rs+++ b/datafusion/expr/src/utils.rs@@ -750,6 +750,74 @@ fn first_span(expr: &Expr) -> Option<Span> {span}+/// Returns an error if `expr` contains a window function call.+///+/// `clause` names where `expr` came from and completes the message+/// `Window function calls are not allowed in {clause}`, for example `WHERE` or+/// `HAVING`. Filters are evaluated before window functions are computed, so a+/// window call in a filter predicate has no physical equivalent and is not+/// valid SQL either (PostgreSQL: `window functions are not allowed in HAVING`).+/// Rejecting it while the logical plan is built gives an error that names the+/// offending call instead of a late physical planning failure.+///+/// [`Filter::try_new`] calls this for every predicate, so the SQL planner and+/// the `DataFrame`/`LogicalPlanBuilder` paths are covered without callers+/// invoking it directly. Window calls inside subqueries of `expr` are not+/// visited and are legal.+///+/// The error is built by [`window_function_not_allowed_err`] with the best+/// effort span of the call (see [`Expr::spans`]) and a generic help message. A+/// caller that knows where the call is in the original query, such as the SQL+/// planner, can build a more precise error with that function directly.+///+/// [`Filter::try_new`]: crate::logical_plan::Filter::try_new+pub fn check_no_window_functions(expr: &Expr, clause: &str) -> Result<()> {+ match first_window_function(expr) {+ None => Ok(()),+ Some(window) => Err(window_function_not_allowed_err(+ window,+ clause,+ first_span(window),+ format!(+ "Compute '{window}' first, in a Window node or an inner query, and filter on its result"+ ),+ )),+ }+}++/// The first window function call in `expr`, if any. Subqueries are not+/// visited.+fn first_window_function(expr: &Expr) -> Option<&Expr> {+ let mut window = None;+ expr.apply(|e| {+ if matches!(e, Expr::WindowFunction(_)) {+ window = Some(e);+ Ok(TreeNodeRecursion::Stop)+ } else {+ Ok(TreeNodeRecursion::Continue)+ }+ })+ .ok()?;+ window+}++/// The planning error for `window`, a window function call that appeared in+/// `clause` where it is not allowed (see [`check_no_window_functions`]).+///+/// The message is `Window function calls are not allowed in {clause}:+/// '{window}'`. The error carries a [`Diagnostic`] with the same message,+/// pointing at `span`, and with `help`.+pub fn window_function_not_allowed_err(+ window: &Expr,+ clause: &str,+ span: Option<Span>,+ help: impl Into<String>,+) -> DataFusionError {+ let message = format!("Window function calls are not allowed in {clause}");+ plan_datafusion_err!("{message}: '{window}'")+ .with_diagnostic(Diagnostic::new_error(message, span).with_help(help, None))+}+/// Collect all deeply nested `Expr::WindowFunction`. They are returned in order of occurrence/// (depth first), with duplicates omitted.pub fn find_window_exprs<'a>(exprs: impl IntoIterator<Item = &'a Expr>) -> Vec<Expr> {@@ -2102,4 +2170,43 @@ mod tests {@"Error during planning: Window function calls cannot be nested: 'sum(a) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING' is nested inside 'sum(sum(a) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'");}++ #[test]+ fn test_check_no_window_functions() -> Result<()> {+ use crate::test::function_stub::sum;+ use insta::assert_snapshot;++ // columns, scalar expressions and aggregates are fine+ check_no_window_functions(&col("a"), "WHERE")?;+ check_no_window_functions(&(col("a") + lit(1)).gt(lit(0)), "WHERE")?;+ check_no_window_functions(&sum(col("a")).gt(lit(0)), "HAVING")?;++ // a bare window call+ let err =+ check_no_window_functions(&sum_over(vec![col("a")]), "WHERE").unwrap_err();+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in WHERE: 'sum(a) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );+ let diag = err.diagnostic().expect("diagnostic");+ assert_snapshot!(+ diag.message,+ @"Window function calls are not allowed in WHERE"+ );+ assert_snapshot!(+ diag.helps[0].message,+ @"Compute 'sum(a) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING' first, in a Window node or an inner query, and filter on its result"+ );++ // a window call over an aggregate, nested below other expressions,+ // names the clause it was given+ let predicate = sum_over(vec![sum(col("a"))]).gt(lit(10)).and(col("b"));+ let err = check_no_window_functions(&predicate, "HAVING").unwrap_err();+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in HAVING: 'sum(sum(a)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ Ok(())+ }}diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rsindex c5b67cde8e..9522b0910c 100644--- a/datafusion/physical-expr/src/planner.rs+++ b/datafusion/physical-expr/src/planner.rs@@ -689,6 +689,16 @@ pub fn create_physical_expr(Arc::clone(schema_field),)))}+ // Window and aggregate functions are evaluated by dedicated plan+ // nodes, never by a physical expression. Reaching this point means+ // the function is in a position where it cannot be evaluated, so+ // report that instead of dumping the internal representation.+ Expr::WindowFunction(_) => plan_err!(+ "Window function '{e}' is not supported in this position. Window functions are supported in the SELECT list, ORDER BY, DISTINCT ON and QUALIFY"+ ),+ Expr::AggregateFunction(_) => plan_err!(+ "Aggregate function '{e}' is not supported in this position. Aggregate functions are supported in the SELECT list, HAVING and ORDER BY of a query with GROUP BY"+ ),other => {not_impl_err!("Physical plan does not support logical expression {other:?}")}diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rsindex f1661028ed..0fa1fc16bf 100644--- a/datafusion/sql/src/expr/mod.rs+++ b/datafusion/sql/src/expr/mod.rs@@ -30,13 +30,14 @@ use sqlparser::ast::{use sqlparser::ast::{Query, Visit, Visitor};use datafusion_common::{- DFSchema, Diagnostic, Result, ScalarValue, Span, internal_datafusion_err,+ DFSchema, Diagnostic, HashMap, Result, ScalarValue, Span, internal_datafusion_err,internal_err, not_impl_err, plan_err,};use datafusion_expr::expr::ScalarFunction;use datafusion_expr::expr::SetQuantifier;use datafusion_expr::expr::{InList, WildcardOptions};+use datafusion_expr::utils::{find_window_exprs, window_function_not_allowed_err};use datafusion_expr::{Between, BinaryExpr, Cast, Expr, ExprSchemable, GetFieldAccess, Like, Literal,Operator, TryCast, lit, when,@@ -137,12 +138,108 @@ impl<S: ContextProvider> Visitor for NullEqualityPredicateVisitor<'_, '_, S> {}}+/// Finds the location of the first window function call in a SQL expression.+/// An identifier that is an alias of a `SELECT` expression containing a window+/// function call counts as well, since aliases are resolved before the window+/// function check (`HAVING total > 0` where `total` is `sum(x) OVER ()`).+/// Subqueries are skipped: window functions are legal there.+struct WindowFunctionSpanVisitor<'a, 'b, S: ContextProvider> {+ sql_to_rel: &'a SqlToRel<'b, S>,+ aliases: &'a HashMap<String, Expr>,+ subquery_depth: usize,+ span: Option<sqlparser::tokenizer::Span>,+}++impl<S: ContextProvider> Visitor for WindowFunctionSpanVisitor<'_, '_, S> {+ type Break = ();++ fn pre_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {+ self.subquery_depth += 1;+ ControlFlow::Continue(())+ }++ fn post_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {+ self.subquery_depth -= 1;+ ControlFlow::Continue(())+ }++ fn pre_visit_expr(&mut self, expr: &SQLExpr) -> ControlFlow<Self::Break> {+ if self.subquery_depth > 0 {+ return ControlFlow::Continue(());+ }+ let found = match expr {+ SQLExpr::Function(function) => function.over.is_some(),+ SQLExpr::Identifier(ident) => {+ let name = self.sql_to_rel.ident_normalizer.normalize(ident.clone());+ self.aliases+ .get(&name)+ .is_some_and(|aliased| !find_window_exprs([aliased]).is_empty())+ }+ _ => false,+ };+ if found {+ self.span = Some(expr.span());+ ControlFlow::Break(())+ } else {+ ControlFlow::Continue(())+ }+ }+}++/// Help for a window function in `WHERE` or `HAVING`: `QUALIFY` is the clause+/// that filters on window function results.+pub(crate) const QUALIFY_HELP: &str = "Move the condition that uses this window function to a QUALIFY clause, which is evaluated after window functions are computed";++/// Returns an error if `expr`, planned from a clause where window functions+/// cannot be evaluated, contains a window function call. `clause` names it in+/// the message, `help` says how to rewrite the query and `span` is the location+/// of the call in the SQL text, see [`SqlToRel::window_function_span`].+///+/// The physical planner rejects such expressions in every position, but+/// without a span or a clause-specific hint. This check exists to give a+/// better error for the common mistakes.+pub(crate) fn reject_window_functions(+ expr: &Expr,+ clause: &str,+ help: &str,+ span: Option<Span>,+) -> Result<()> {+ match find_window_exprs([expr]).into_iter().next() {+ None => Ok(()),+ Some(window) => Err(window_function_not_allowed_err(&window, clause, span, help)),+ }+}+impl<S: ContextProvider> SqlToRel<'_, S> {pub(crate) fn warn_on_null_equality_predicate(&self, predicate: &SQLExpr) {let mut visitor = NullEqualityPredicateVisitor::new(self);let _ = predicate.visit(&mut visitor);}+ /// The location in the SQL text of the first window function call in+ /// `expr`, or of the first identifier that is an alias (in `aliases`) of a+ /// `SELECT` expression containing one. Falls back to the location of the+ /// whole of `expr`, and returns `None` only if the parser recorded no+ /// spans.+ ///+ /// Used to point the [`Diagnostic`] for a window function in `WHERE` or+ /// `HAVING` at the offending call, which the planned [`Expr`] cannot do:+ /// only columns carry spans there.+ pub(crate) fn window_function_span(+ &self,+ expr: &SQLExpr,+ aliases: &HashMap<String, Expr>,+ ) -> Option<Span> {+ let mut visitor = WindowFunctionSpanVisitor {+ sql_to_rel: self,+ aliases,+ subquery_depth: 0,+ span: None,+ };+ let _ = expr.visit(&mut visitor);+ Span::try_from_sqlparser_span(visitor.span.unwrap_or_else(|| expr.span()))+ }+pub(crate) fn sql_expr_to_logical_expr_with_alias(&self,sql: SQLExprWithAlias,diff --git a/datafusion/sql/src/relation/join.rs b/datafusion/sql/src/relation/join.rsindex 73c2f05504..d8085133e5 100644--- a/datafusion/sql/src/relation/join.rs+++ b/datafusion/sql/src/relation/join.rs@@ -15,14 +15,20 @@// specific language governing permissions and limitations// under the License.+use crate::expr::reject_window_functions;use crate::planner::{ContextProvider, PlannerContext, SqlToRel};-use datafusion_common::{Column, Result, not_impl_err, plan_datafusion_err, plan_err};+use datafusion_common::{+ Column, HashMap, Result, not_impl_err, plan_datafusion_err, plan_err,+};use datafusion_expr::{JoinType, LogicalPlan, LogicalPlanBuilder};use sqlparser::ast::{Join, JoinConstraint, JoinOperator, ObjectName, TableFactor, TableWithJoins,};use std::collections::HashSet;+const JOIN_ON_HELP: &str =+ "Compute the window function in a subquery and join on its result";+impl<S: ContextProvider> SqlToRel<'_, S> {pub(crate) fn plan_table_with_joins(&self,@@ -126,7 +132,9 @@ impl<S: ContextProvider> SqlToRel<'_, S> {match constraint {JoinConstraint::On(sql_on) => {+ let window_span = self.window_function_span(&sql_on, &HashMap::new());let on = self.sql_to_expr(sql_on, &join_schema, planner_context)?;+ reject_window_functions(&on, "JOIN ON", JOIN_ON_HELP, window_span)?;LogicalPlanBuilder::from(left).asof_join_on(right, Some(on), match_condition)?.build()@@ -188,7 +196,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let join_schema = left.schema().join(right.schema())?;// parse ON expressionself.warn_on_null_equality_predicate(&sql_expr);+ let window_span = self.window_function_span(&sql_expr, &HashMap::new());let expr = self.sql_to_expr(sql_expr, &join_schema, planner_context)?;+ // A join condition is evaluated before window functions are+ // computed, so they may not appear in it+ reject_window_functions(&expr, "JOIN ON", JOIN_ON_HELP, window_span)?;LogicalPlanBuilder::from(left).join_on(right, join_type, Some(expr))?.build()diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex bbd9d203eb..86916c5693 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -19,6 +19,7 @@ use std::collections::HashSet;use std::ops::ControlFlow;use std::sync::Arc;+use crate::expr::{QUALIFY_HELP, reject_window_functions};use crate::planner::{ContextProvider, PlannerContext, SqlToRel};use crate::query::to_order_by_exprs_with_select;use crate::utils::{@@ -32,7 +33,9 @@ use crate::utils::{use arrow::datatypes::DataType;use datafusion_common::error::DataFusionErrorBuilder;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};-use datafusion_common::{Column, DFSchema, DFSchemaRef, Result, not_impl_err, plan_err};+use datafusion_common::{+ Column, DFSchema, DFSchemaRef, HashMap, Result, not_impl_err, plan_err,+};use datafusion_common::{NullHandling, RecursionUnnestOption, UnnestOptions};use datafusion_expr::ExprSchemable;use datafusion_expr::builder::get_struct_unnested_columns;@@ -202,6 +205,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {.having.map::<Result<Expr>, _>(|having_expr| {self.warn_on_null_equality_predicate(&having_expr);+ let window_span = self.window_function_span(&having_expr, &alias_map);let having_expr = self.sql_expr_to_logical_expr(having_expr,&combined_schema,@@ -221,6 +225,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// SELECT c1, MAX(c2) AS m FROM t GROUP BY c1 HAVING MAX(c2) > 10;//let having_expr = resolve_aliases_to_exprs(having_expr, &alias_map)?;+ // HAVING is evaluated before window functions are computed, so+ // they may not appear there (checked after alias resolution so+ // that an alias of a window function is rejected too)+ reject_window_functions(+ &having_expr,+ "HAVING",+ QUALIFY_HELP,+ window_span,+ )?;let having_expr = normalize_col(having_expr, &projected_plan)?;let (having_expr, _) =having_expr.infer_placeholder_types(&combined_schema)?;@@ -233,21 +246,31 @@ impl<S: ContextProvider> SqlToRel<'_, S> {exprs.into_iter().map(|e| {- let group_by_expr = self.sql_expr_to_logical_expr(- e,- &combined_schema,- planner_context,- )?;-// Aliases from the projection can conflict with same-named expressions in the inputlet mut alias_map = alias_map.clone();for f in base_plan.schema().fields() {alias_map.remove(f.name());}+ let window_span = self.window_function_span(&e, &alias_map);++ let group_by_expr = self.sql_expr_to_logical_expr(+ e,+ &combined_schema,+ planner_context,+ )?;let group_by_expr =resolve_aliases_to_exprs(group_by_expr, &alias_map)?;let group_by_expr =resolve_positions_to_exprs(group_by_expr, &select_exprs)?;+ // Window functions are computed after grouping, so they may+ // not be grouped on (checked after aliases and positions are+ // resolved so `GROUP BY rn` / `GROUP BY 1` are rejected too)+ reject_window_functions(+ &group_by_expr,+ "GROUP BY",+ "Compute the window function in a subquery and group by its result",+ window_span,+ )?;let group_by_expr = normalize_col(group_by_expr, &projected_plan)?;self.validate_schema_satisfies_exprs(base_plan.schema(),@@ -885,6 +908,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let fallback_schemas = plan.fallback_normalize_schemas();self.warn_on_null_equality_predicate(&predicate_expr);+ let window_span =+ self.window_function_span(&predicate_expr, &HashMap::new());let filter_expr =self.sql_to_expr(predicate_expr, plan.schema(), planner_context)?;@@ -896,6 +921,14 @@ impl<S: ContextProvider> SqlToRel<'_, S> {"Aggregate functions are not allowed in the WHERE clause. Consider using HAVING instead");}+ // WHERE is evaluated before window functions are computed, so+ // they may not appear there either+ reject_window_functions(+ &filter_expr,+ "WHERE",+ QUALIFY_HELP,+ window_span,+ )?;let mut using_columns = HashSet::new();expr_to_columns(&filter_expr, &mut using_columns)?;diff --git a/datafusion/sql/src/values.rs b/datafusion/sql/src/values.rsindex a1df1fe1b1..2f15785520 100644--- a/datafusion/sql/src/values.rs+++ b/datafusion/sql/src/values.rs@@ -17,8 +17,9 @@use std::sync::Arc;+use crate::expr::reject_window_functions;use crate::planner::{ContextProvider, PlannerContext, SqlToRel};-use datafusion_common::{DFSchema, Result, not_impl_err};+use datafusion_common::{DFSchema, HashMap, Result, not_impl_err};use datafusion_expr::{LogicalPlan, LogicalPlanBuilder};use sqlparser::ast::Values as SQLValues;@@ -45,7 +46,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {.map(|row| {row.content.into_iter()- .map(|v| self.sql_to_expr(v, &empty_schema, planner_context))+ .map(|v| {+ let window_span = self.window_function_span(&v, &HashMap::new());+ let expr = self.sql_to_expr(v, &empty_schema, planner_context)?;+ // A VALUES row has no input rows to compute a window over+ reject_window_functions(+ &expr,+ "VALUES",+ "Compute the window function in a SELECT over a table or subquery instead",+ window_span,+ )?;+ Ok(expr)+ }).collect::<Result<Vec<_>>>()}).collect::<Result<Vec<_>>>()?;diff --git a/datafusion/sql/tests/cases/diagnostic.rs b/datafusion/sql/tests/cases/diagnostic.rsindex c385393bed..2a6983169c 100644--- a/datafusion/sql/tests/cases/diagnostic.rs+++ b/datafusion/sql/tests/cases/diagnostic.rs@@ -708,3 +708,86 @@ fn test_nested_window_function() -> Result<()> {assert_eq!(diag.span, Some(spans["a"]));Ok(())}++#[test]+fn test_window_function_in_where() -> Result<()> {+ // https://github.com/apache/datafusion/issues/4610+ //+ // The span is taken from the SQL text and covers the window function call.+ // sqlparser's span for a function call ends at its last argument, so the+ // `) OVER ()` part is not included.+ let query = "SELECT id FROM person WHERE /*a*/sum(age/*a*/) OVER () > 0";+ let spans = get_spans(query);+ let diag = do_query(query);+ assert_snapshot!(diag.message, @"Window function calls are not allowed in WHERE");+ assert_eq!(diag.span, Some(spans["a"]));+ assert_snapshot!(+ diag.helps[0].message,+ @"Move the condition that uses this window function to a QUALIFY clause, which is evaluated after window functions are computed"+ );+ Ok(())+}++#[test]+fn test_window_function_in_where_after_subquery() -> Result<()> {+ // A window function inside a subquery is legal and must not be the one the+ // span points at+ let query = "SELECT id FROM person WHERE id IN (SELECT id FROM person QUALIFY sum(age) OVER () > 0) AND /*a*/sum(age/*a*/) OVER () > 0";+ let spans = get_spans(query);+ let diag = do_query(query);+ assert_snapshot!(diag.message, @"Window function calls are not allowed in WHERE");+ assert_eq!(diag.span, Some(spans["a"]));+ Ok(())+}++#[test]+fn test_window_function_in_having() -> Result<()> {+ // https://github.com/apache/datafusion/issues/4610+ let query = "SELECT first_name FROM person GROUP BY first_name HAVING /*a*/sum(sum(age/*a*/)) OVER () > 0";+ let spans = get_spans(query);+ let diag = do_query(query);+ assert_snapshot!(diag.message, @"Window function calls are not allowed in HAVING");+ assert_eq!(diag.span, Some(spans["a"]));+ Ok(())+}++#[test]+fn test_window_function_in_join_on() -> Result<()> {+ let query = "SELECT p.id FROM person p JOIN person q ON p.id = q.id AND /*a*/sum(p.age/*a*/) OVER () > 0";+ let spans = get_spans(query);+ let diag = do_query(query);+ assert_snapshot!(diag.message, @"Window function calls are not allowed in JOIN ON");+ assert_eq!(diag.span, Some(spans["a"]));+ assert_snapshot!(+ diag.helps[0].message,+ @"Compute the window function in a subquery and join on its result"+ );+ Ok(())+}++#[test]+fn test_window_function_in_group_by() -> Result<()> {+ let query =+ "SELECT first_name FROM person GROUP BY first_name, /*a*/sum(age/*a*/) OVER ()";+ let spans = get_spans(query);+ let diag = do_query(query);+ assert_snapshot!(diag.message, @"Window function calls are not allowed in GROUP BY");+ assert_eq!(diag.span, Some(spans["a"]));+ assert_snapshot!(+ diag.helps[0].message,+ @"Compute the window function in a subquery and group by its result"+ );+ Ok(())+}++#[test]+fn test_window_function_alias_in_having() -> Result<()> {+ // An alias of a window function is resolved before the check; the span+ // points at the alias+ let query = "SELECT first_name, sum(sum(age)) OVER () AS total FROM person GROUP BY first_name HAVING /*a*/total/*a*/ > 0";+ let spans = get_spans(query);+ let diag = do_query(query);+ assert_snapshot!(diag.message, @"Window function calls are not allowed in HAVING");+ assert_eq!(diag.span, Some(spans["a"]));+ Ok(())+}diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rsindex 1fadca5eff..2bd06ee4ba 100644--- a/datafusion/sql/tests/sql_integration.rs+++ b/datafusion/sql/tests/sql_integration.rs@@ -1972,6 +1972,147 @@ fn select_nested_window_function() {);}+#[test]+fn select_window_function_in_where() {+ // https://github.com/apache/datafusion/issues/4610+ let err = logical_plan("SELECT id FROM person WHERE sum(age) OVER () > 0")+ .expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in WHERE: 'sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ // a window function inside a subquery of the predicate is legal+ let plan = logical_plan(+ "SELECT id FROM person WHERE id IN (SELECT id FROM person QUALIFY sum(age) OVER () > 0)",+ )+ .unwrap();+ assert_snapshot!(+ plan,+ @r"+ Projection: person.id+ Filter: person.id IN (<subquery>)+ Subquery:+ Projection: person.id+ Filter: sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING > Int64(0)+ WindowAggr: windowExpr=[[sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]+ TableScan: person+ TableScan: person+ "+ );+}++#[test]+fn select_window_function_in_having() {+ // https://github.com/apache/datafusion/issues/4610+ let err = logical_plan(+ "SELECT state, sum(count(age)) OVER () AS total FROM person GROUP BY state HAVING sum(count(age)) OVER () > 0",+ )+ .expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in HAVING: 'sum(count(person.age)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ // an alias of a window function is resolved before the check+ let err = logical_plan(+ "SELECT state, sum(count(age)) OVER () AS total FROM person GROUP BY state HAVING total > 0",+ )+ .expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in HAVING: 'sum(count(person.age)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ // the same window function is legal in the SELECT list and in QUALIFY+ let plan = logical_plan(+ "SELECT state, sum(count(age)) OVER () AS total FROM person GROUP BY state QUALIFY sum(count(age)) OVER () > 0",+ )+ .unwrap();+ assert_snapshot!(+ plan,+ @r"+ Projection: person.state, sum(count(person.age)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS total+ Filter: sum(count(person.age)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING > Int64(0)+ WindowAggr: windowExpr=[[sum(count(person.age)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]+ Aggregate: groupBy=[[person.state]], aggr=[[count(person.age)]]+ TableScan: person+ "+ );+}++#[test]+fn select_window_function_in_join_on() {+ // https://github.com/apache/datafusion/issues/4610+ let err = logical_plan(+ "SELECT p.id FROM person p JOIN person q ON p.id = q.id AND sum(p.age) OVER () > 0",+ )+ .expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in JOIN ON: 'sum(p.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ // a window function in a joined subquery is legal+ let plan = logical_plan(+ "SELECT p.id FROM person p JOIN (SELECT id FROM person QUALIFY sum(age) OVER () > 0) q ON p.id = q.id",+ )+ .unwrap();+ assert_snapshot!(+ plan,+ @r"+ Projection: p.id+ Inner Join: Filter: p.id = q.id+ SubqueryAlias: p+ TableScan: person+ SubqueryAlias: q+ Projection: person.id+ Filter: sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING > Int64(0)+ WindowAggr: windowExpr=[[sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]]+ TableScan: person+ "+ );+}++#[test]+fn select_window_function_in_group_by() {+ // https://github.com/apache/datafusion/issues/4610+ let err = logical_plan("SELECT count(*) FROM person GROUP BY sum(age) OVER ()")+ .expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in GROUP BY: 'sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ // through an alias of a select expression+ let err =+ logical_plan("SELECT sum(age) OVER () AS w, count(*) FROM person GROUP BY w")+ .expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in GROUP BY: 'sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );++ // through the position of a select expression+ let err = logical_plan("SELECT sum(age) OVER (), count(*) FROM person GROUP BY 1")+ .expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in GROUP BY: 'sum(person.age) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );+}++#[test]+fn values_window_function() {+ // https://github.com/apache/datafusion/issues/4610+ let err =+ logical_plan("VALUES (1, sum(2) OVER ())").expect_err("query should have failed");+ assert_snapshot!(+ err.strip_backtrace(),+ @"Error during planning: Window function calls are not allowed in VALUES: 'sum(Int64(2)) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'"+ );+}+#[test]fn select_aggregate_inside_window_function() {// an aggregate as the argument of a window function is legal: the windowdiff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex bd3eb6ba39..3d6620fc8a 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -7093,3 +7093,114 @@ SELECT last_value() OVER () FROM (VALUES (1), (2), (3)) t(x)statement error DataFusion error: Error during planning: 'nth_value' does not support zero argumentsSELECT nth_value() OVER () FROM (VALUES (1), (2), (3)) t(x)++# Window functions are not allowed in WHERE or HAVING: both are evaluated before+# window functions are computed, so they are rejected during planning+# issue: https://github.com/apache/datafusion/issues/4610+statement ok+CREATE TABLE window_in_filter_t(a INT, b INT) AS VALUES (1, 1), (2, 2);++statement error DataFusion error: Error during planning: Window function calls are not allowed in HAVING: 'sum\(count\(Int64\(1\)\) AS count\(\*\)\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT a, count(*) AS cnt, sum(count(*)) OVER () AS total+FROM window_in_filter_t+GROUP BY a+HAVING sum(count(*)) OVER () >= 10;++# ... also through an alias of the window function+statement error DataFusion error: Error during planning: Window function calls are not allowed in HAVING: 'sum\(count\(Int64\(1\)\) AS count\(\*\)\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT a, count(*) AS cnt, sum(count(*)) OVER () AS total+FROM window_in_filter_t+GROUP BY a+HAVING total >= 10;++statement error DataFusion error: Error during planning: Window function calls are not allowed in HAVING: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT a FROM window_in_filter_t GROUP BY a HAVING row_number() OVER () = 1;++statement error DataFusion error: Error during planning: Window function calls are not allowed in WHERE: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT a FROM window_in_filter_t WHERE row_number() OVER () = 1;++# The same window function is legal in the SELECT list ...+query III+SELECT a, count(*) AS cnt, sum(count(*)) OVER () AS total+FROM window_in_filter_t+GROUP BY a+ORDER BY a;+----+1 1 2+2 1 2++# ... in QUALIFY, which is the clause for filtering on a window function ...+query III+SELECT a, count(*) AS cnt, sum(count(*)) OVER () AS total+FROM window_in_filter_t+GROUP BY a+QUALIFY sum(count(*)) OVER () >= 2+ORDER BY a;+----+1 1 2+2 1 2++# ... in the WHERE of an outer query ...+query III+SELECT * FROM (+ SELECT a, count(*) AS cnt, sum(count(*)) OVER () AS total+ FROM window_in_filter_t+ GROUP BY a+) s+WHERE total >= 2+ORDER BY a;+----+1 1 2+2 1 2++# ... and inside a subquery of a WHERE predicate+query I+SELECT a FROM window_in_filter_t+WHERE a IN (SELECT a FROM window_in_filter_t QUALIFY row_number() OVER (ORDER BY a) = 1);+----+1++# Window functions are rejected during planning in JOIN ON, GROUP BY and VALUES+# as well+statement error DataFusion error: Error during planning: Window function calls are not allowed in JOIN ON: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT * FROM window_in_filter_t x JOIN window_in_filter_t y ON x.a = y.a AND row_number() OVER () = 1;++statement error DataFusion error: Error during planning: Window function calls are not allowed in GROUP BY: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT count(*) FROM window_in_filter_t GROUP BY row_number() OVER ();++# ... also through an alias or the position of a select expression+statement error DataFusion error: Error during planning: Window function calls are not allowed in GROUP BY: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT row_number() OVER () AS rn, count(*) FROM window_in_filter_t GROUP BY rn;++statement error DataFusion error: Error during planning: Window function calls are not allowed in GROUP BY: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+SELECT row_number() OVER (), count(*) FROM window_in_filter_t GROUP BY 1;++statement error DataFusion error: Error during planning: Window function calls are not allowed in VALUES: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+VALUES (row_number() OVER ());++statement error DataFusion error: Error during planning: Window function calls are not allowed in VALUES: 'row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'+INSERT INTO window_in_filter_t VALUES (row_number() OVER (), 1);++# A window function in a joined subquery is legal+query II+SELECT x.a, y.total FROM window_in_filter_t x+JOIN (SELECT a, sum(b) OVER () AS total FROM window_in_filter_t) y ON x.a = y.a+ORDER BY x.a;+----+1 3+2 3++# Positions with no dedicated check are reported by the physical planner, which+# names the function instead of dumping its internal representation+statement error DataFusion error: This feature is not implemented: Unsupported LIMIT expression: CAST\(row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS Int64\)+SELECT a FROM window_in_filter_t LIMIT row_number() OVER ();++statement error DataFusion error: This feature is not implemented: Unsupported OFFSET expression: CAST\(row_number\(\) ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING AS Int64\)+SELECT a FROM window_in_filter_t OFFSET row_number() OVER ();++# ... and the same for an aggregate function+statement error DataFusion error: This feature is not implemented: Unsupported LIMIT expression: count\(Int64\(1\)\) AS count\(\*\)+SELECT a FROM window_in_filter_t LIMIT count(*);++statement ok+DROP TABLE window_in_filter_t;