Apache DataFusion SQL Query Engine
Resolve named windows referenced inside OVER (...) (#26400)
Qi Zhu6131a9813af3
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex 8b7718ca00..34907c489c 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -34,7 +34,8 @@ use arrow::datatypes::DataType;use datafusion_common::error::DataFusionErrorBuilder;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion_common::{- Column, DFSchema, DFSchemaRef, HashMap, Result, not_impl_err, plan_err,+ Column, DFSchema, DFSchemaRef, DataFusionError, HashMap, Result, not_impl_err,+ plan_err,};use datafusion_common::{NullHandling, RecursionUnnestOption, UnnestOptions};use datafusion_expr::ExprSchemable;@@ -55,9 +56,9 @@ use datafusion_expr::{use indexmap::IndexMap;use sqlparser::ast::{- Distinct, Expr as SQLExpr, GroupByExpr, NamedWindowExpr, OrderBy,- SelectItemQualifiedWildcardKind, WildcardAdditionalOptions, WindowType,- visit_expressions_mut,+ Distinct, Expr as SQLExpr, GroupByExpr, Ident, NamedWindowExpr, OrderBy, Query,+ SelectItemQualifiedWildcardKind, VisitMut, VisitorMut, WildcardAdditionalOptions,+ WindowSpec, WindowType,};use sqlparser::ast::{NamedWindowDefinition, Select, SelectItem, TableWithJoins};@@ -99,7 +100,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {pub(super) fn select_to_plan(&self,mut select: Select,- query_order_by: Option<OrderBy>,+ mut query_order_by: Option<OrderBy>,planner_context: &mut PlannerContext,) -> Result<LogicalPlan> {// Check for unsupported syntax first@@ -131,6 +132,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// Handle named windows before processing the projection expressioncheck_conflicting_windows(&select.named_window)?;self.match_window_definitions(&mut select.projection, &select.named_window)?;+ self.match_window_definitions(&mut select.qualify, &select.named_window)?;+ self.match_window_definitions(&mut query_order_by, &select.named_window)?;// Process the SELECT expressionslet select_exprs = self.prepare_select_exprs(@@ -1420,58 +1423,146 @@ impl<S: ContextProvider> SqlToRel<'_, S> {})}- // If the projection is done over a named window, that window- // name must be defined. Otherwise, it gives an error.+ /// Replaces every window that references a named window, as in `OVER w`+ /// or `OVER (w ...)`, with the resolved window specification.+ ///+ /// Subqueries are skipped: each resolves names against its own `WINDOW`+ /// clause when it is planned.fn match_window_definitions(&self,- projection: &mut [SelectItem],+ node: &mut impl VisitMut,named_windows: &[NamedWindowDefinition],) -> Result<()> {let named_windows: Vec<(&NamedWindowDefinition, String)> = named_windows.iter().map(|w| (w, self.ident_normalizer.normalize(w.0.clone()))).collect();- for proj in projection.iter_mut() {- if let SelectItem::ExprWithAlias { expr, alias: _ }- | SelectItem::UnnamedExpr(expr) = proj- {- let mut err = None;- let _ = visit_expressions_mut(expr, |expr| {- if let SQLExpr::Function(f) = expr- && let Some(WindowType::NamedWindow(ident)) = &f.over- {- let normalized_ident =- self.ident_normalizer.normalize(ident.clone());- for (- NamedWindowDefinition(_, window_expr),- normalized_window_ident,- ) in named_windows.iter()- {- if normalized_ident.eq(normalized_window_ident) {- f.over = Some(match window_expr {- NamedWindowExpr::NamedWindow(ident) => {- WindowType::NamedWindow(ident.clone())- }- NamedWindowExpr::WindowSpec(spec) => {- WindowType::WindowSpec(spec.clone())- }- })- }- }- // All named windows must be defined with a WindowSpec.- if let Some(WindowType::NamedWindow(ident)) = &f.over {- err = Some(plan_err!("The window {ident} is not defined!"));- return ControlFlow::Break(());- }- }- ControlFlow::Continue(())- });- if let Some(err) = err {- return err;- }+ let mut resolver = NamedWindowResolver {+ planner: self,+ named_windows: &named_windows,+ query_depth: 0,+ };+ match node.visit(&mut resolver) {+ ControlFlow::Break(e) => Err(e),+ ControlFlow::Continue(()) => Ok(()),+ }+ }++ /// Returns the window specification of the named window `ident`.+ ///+ /// A window definition may only reference windows defined before it, so+ /// references cannot form a cycle.+ fn resolve_named_window(+ &self,+ ident: &Ident,+ named_windows: &[(&NamedWindowDefinition, String)],+ ) -> Result<WindowSpec> {+ let normalized_ident = self.ident_normalizer.normalize(ident.clone());+ let Some(position) = named_windows+ .iter()+ .position(|(_, name)| *name == normalized_ident)+ else {+ return plan_err!("The window {ident} is not defined!");+ };+ let (NamedWindowDefinition(_, window_expr), _) = named_windows[position];+ let earlier_windows = &named_windows[..position];+ match window_expr {+ NamedWindowExpr::NamedWindow(base) => {+ self.resolve_named_window(base, earlier_windows)+ }+ NamedWindowExpr::WindowSpec(spec) => {+ self.resolve_window_spec(spec, earlier_windows)+ }+ }+ }++ /// Resolves a window specification that may reference a named window, as+ /// in `OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)`.+ ///+ /// Following the SQL standard and PostgreSQL, the specification copies+ /// the referenced window's `PARTITION BY` and `ORDER BY`, may add an+ /// `ORDER BY` only when the referenced window has none, and may add a+ /// frame only when the referenced window has none.+ fn resolve_window_spec(+ &self,+ spec: &WindowSpec,+ named_windows: &[(&NamedWindowDefinition, String)],+ ) -> Result<WindowSpec> {+ let Some(base_name) = &spec.window_name else {+ return Ok(spec.clone());+ };+ let base = self.resolve_named_window(base_name, named_windows)?;+ if !spec.partition_by.is_empty() {+ return plan_err!(+ "Cannot override PARTITION BY clause of window {base_name}"+ );+ }+ if !spec.order_by.is_empty() && !base.order_by.is_empty() {+ return plan_err!("Cannot override ORDER BY clause of window {base_name}");+ }+ if base.window_frame.is_some() {+ return plan_err!(+ "Cannot copy window {base_name} because it has a frame clause"+ );+ }+ let order_by = if spec.order_by.is_empty() {+ base.order_by+ } else {+ spec.order_by.clone()+ };+ Ok(WindowSpec {+ window_name: None,+ partition_by: base.partition_by,+ order_by,+ window_frame: spec.window_frame.clone(),+ })+ }+}++/// Resolves named window references in the expressions of a single `SELECT`.+struct NamedWindowResolver<'a, 'b, S: ContextProvider> {+ planner: &'a SqlToRel<'b, S>,+ named_windows: &'a [(&'a NamedWindowDefinition, String)],+ /// Number of subqueries enclosing the expression being visited.+ query_depth: usize,+}++impl<S: ContextProvider> VisitorMut for NamedWindowResolver<'_, '_, S> {+ type Break = DataFusionError;++ fn pre_visit_query(&mut self, _query: &mut Query) -> ControlFlow<Self::Break> {+ self.query_depth += 1;+ ControlFlow::Continue(())+ }++ fn post_visit_query(&mut self, _query: &mut Query) -> ControlFlow<Self::Break> {+ self.query_depth -= 1;+ ControlFlow::Continue(())+ }++ fn post_visit_expr(&mut self, expr: &mut SQLExpr) -> ControlFlow<Self::Break> {+ if self.query_depth > 0 {+ return ControlFlow::Continue(());+ }+ let SQLExpr::Function(f) = expr else {+ return ControlFlow::Continue(());+ };+ let resolved = match &f.over {+ Some(WindowType::NamedWindow(ident)) => {+ self.planner.resolve_named_window(ident, self.named_windows)+ }+ Some(WindowType::WindowSpec(spec)) if spec.window_name.is_some() => {+ self.planner.resolve_window_spec(spec, self.named_windows)+ }+ _ => return ControlFlow::Continue(()),+ };+ match resolved {+ Ok(spec) => {+ f.over = Some(WindowType::WindowSpec(spec));+ ControlFlow::Continue(())}+ Err(e) => ControlFlow::Break(e),}- Ok(())}}diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex cea8060c13..b224308303 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -3481,6 +3481,157 @@ SELECTORDER BY C3LIMIT 5+# A window that references a named window inherits its PARTITION BY and ORDER BY+statement ok+CREATE TABLE named_window_prices(ticker VARCHAR, ts INT, price DOUBLE) AS VALUES+ ('BTC', 1, 100.0), ('BTC', 2, 105.0), ('BTC', 3, 102.0),+ ('DOGE', 1, 0.1), ('DOGE', 2, 0.2), ('DOGE', 3, 0.15);++# Adding a frame to a named window keeps its PARTITION BY and ORDER BY+query TIRRR+SELECT ticker, ts, price,+ MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS named_plus_frame,+ MAX(price) OVER (PARTITION BY ticker ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS inline+FROM named_window_prices+WINDOW w AS (PARTITION BY ticker ORDER BY ts)+ORDER BY ticker, ts+----+BTC 1 100 100 100+BTC 2 105 105 105+BTC 3 102 105 105+DOGE 1 0.1 0.1 0.1+DOGE 2 0.2 0.2 0.2+DOGE 3 0.15 0.2 0.2++# Adding an ORDER BY to a named window that only partitions+query TII+SELECT ticker, ts,+ ROW_NUMBER() OVER (w ORDER BY ts DESC) AS rn+FROM named_window_prices+WINDOW w AS (PARTITION BY ticker)+ORDER BY ticker, ts+----+BTC 1 3+BTC 2 2+BTC 3 1+DOGE 1 3+DOGE 2 2+DOGE 3 1++# A window definition can reference an earlier one+query TIR+SELECT ticker, ts,+ SUM(price) OVER w2 AS running+FROM named_window_prices+WINDOW w1 AS (PARTITION BY ticker ORDER BY ts),+ w2 AS (w1 ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)+ORDER BY ticker, ts+----+BTC 1 100+BTC 2 205+BTC 3 307+DOGE 1 0.1+DOGE 2 0.3+DOGE 3 0.45++# Referencing a named window inside parentheses without adding anything+query TII+SELECT ticker, ts, ROW_NUMBER() OVER (w) AS rn+FROM named_window_prices+WINDOW w AS (PARTITION BY ticker ORDER BY ts)+ORDER BY ticker, ts+----+BTC 1 1+BTC 2 2+BTC 3 3+DOGE 1 1+DOGE 2 2+DOGE 3 3++statement error DataFusion error: Error during planning: The window w is not defined!+SELECT MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)+FROM named_window_prices++statement error DataFusion error: Error during planning: Cannot override PARTITION BY clause of window w+SELECT MAX(price) OVER (w PARTITION BY ts)+FROM named_window_prices+WINDOW w AS (ORDER BY ts)++statement error DataFusion error: Error during planning: Cannot override ORDER BY clause of window w+SELECT MAX(price) OVER (w ORDER BY price)+FROM named_window_prices+WINDOW w AS (PARTITION BY ticker ORDER BY ts)++statement error DataFusion error: Error during planning: Cannot copy window w because it has a frame clause+SELECT MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)+FROM named_window_prices+WINDOW w AS (PARTITION BY ticker ORDER BY ts ROWS UNBOUNDED PRECEDING)++# A window definition cannot reference a later one+statement error DataFusion error: Error during planning: The window w2 is not defined!+SELECT SUM(price) OVER w1+FROM named_window_prices+WINDOW w1 AS (w2 ROWS UNBOUNDED PRECEDING),+ w2 AS (PARTITION BY ticker ORDER BY ts)++# Named windows are resolved in QUALIFY+query TIR+SELECT ticker, ts, price+FROM named_window_prices+QUALIFY MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) = price+WINDOW w AS (PARTITION BY ticker ORDER BY ts)+ORDER BY ticker, ts+----+BTC 1 100+BTC 2 105+DOGE 1 0.1+DOGE 2 0.2++query TI+SELECT ticker, ts+FROM named_window_prices+QUALIFY ROW_NUMBER() OVER w = 1+WINDOW w AS (PARTITION BY ticker ORDER BY ts)+ORDER BY ticker+----+BTC 1+DOGE 1++# Named windows are resolved in ORDER BY+query TI+SELECT ticker, ts+FROM named_window_prices+WINDOW w AS (PARTITION BY ticker ORDER BY ts)+ORDER BY ROW_NUMBER() OVER (w ROWS UNBOUNDED PRECEDING) DESC, ticker+----+BTC 3+DOGE 3+BTC 2+DOGE 2+BTC 1+DOGE 1++# A subquery resolves window names against its own WINDOW clause+query I+SELECT (SELECT MAX(rn) FROM (+ SELECT ROW_NUMBER() OVER (w ROWS UNBOUNDED PRECEDING) AS rn+ FROM named_window_prices+ WINDOW w AS (PARTITION BY ticker ORDER BY ts)))+FROM named_window_prices+WINDOW w AS (ORDER BY ts)+LIMIT 1+----+3++statement error DataFusion error: Error during planning: The window w is not defined!+SELECT (SELECT MAX(rn) FROM (+ SELECT ROW_NUMBER() OVER w AS rn FROM named_window_prices))+FROM named_window_prices+WINDOW w AS (ORDER BY ts)++statement ok+DROP TABLE named_window_prices;+# Create a source where there is multiple orderings.statement okCREATE EXTERNAL TABLE multiple_ordered_table (