Skip to content

Apache DataFusion SQL Query Engine

Resolve named windows referenced inside OVER (...) (#26400)

Qi Zhu6131a9813af3
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs
index 8b7718ca00..34907c489c 100644
--- a/datafusion/sql/src/select.rs
+++ b/datafusion/sql/src/select.rs
@@ -34,7 +34,8 @@ use arrow::datatypes::DataType;
use datafusion_common::error::DataFusionErrorBuilder;
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
use datafusion_common::{
- Column, DFSchema, DFSchemaRef, HashMap, Result, not_impl_err, plan_err,
+ Column, DFSchema, DFSchemaRef, DataFusionError, HashMap, Result, not_impl_err,
+ plan_err,
};
use datafusion_common::{NullHandling, RecursionUnnestOption, UnnestOptions};
use datafusion_expr::ExprSchemable;
@@ -55,9 +56,9 @@ use datafusion_expr::{
use indexmap::IndexMap;
use sqlparser::ast::{
- Distinct, Expr as SQLExpr, GroupByExpr, NamedWindowExpr, OrderBy,
- SelectItemQualifiedWildcardKind, WildcardAdditionalOptions, WindowType,
- visit_expressions_mut,
+ Distinct, Expr as SQLExpr, GroupByExpr, Ident, NamedWindowExpr, OrderBy, Query,
+ SelectItemQualifiedWildcardKind, VisitMut, VisitorMut, WildcardAdditionalOptions,
+ WindowSpec, WindowType,
};
use sqlparser::ast::{NamedWindowDefinition, Select, SelectItem, TableWithJoins};
@@ -99,7 +100,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
pub(super) fn select_to_plan(
&self,
mut select: Select,
- query_order_by: Option<OrderBy>,
+ mut query_order_by: Option<OrderBy>,
planner_context: &mut PlannerContext,
) -> Result<LogicalPlan> {
// Check for unsupported syntax first
@@ -131,6 +132,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// Handle named windows before processing the projection expression
check_conflicting_windows(&select.named_window)?;
self.match_window_definitions(&mut select.projection, &select.named_window)?;
+ self.match_window_definitions(&mut select.qualify, &select.named_window)?;
+ self.match_window_definitions(&mut query_order_by, &select.named_window)?;
// Process the SELECT expressions
let select_exprs = self.prepare_select_exprs(
@@ -1420,58 +1423,146 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
})
}
- // If the projection is done over a named window, that window
- // name must be defined. Otherwise, it gives an error.
+ /// Replaces every window that references a named window, as in `OVER w`
+ /// or `OVER (w ...)`, with the resolved window specification.
+ ///
+ /// Subqueries are skipped: each resolves names against its own `WINDOW`
+ /// clause when it is planned.
fn match_window_definitions(
&self,
- projection: &mut [SelectItem],
+ node: &mut impl VisitMut,
named_windows: &[NamedWindowDefinition],
) -> Result<()> {
let named_windows: Vec<(&NamedWindowDefinition, String)> = named_windows
.iter()
.map(|w| (w, self.ident_normalizer.normalize(w.0.clone())))
.collect();
- for proj in projection.iter_mut() {
- if let SelectItem::ExprWithAlias { expr, alias: _ }
- | SelectItem::UnnamedExpr(expr) = proj
- {
- let mut err = None;
- let _ = visit_expressions_mut(expr, |expr| {
- if let SQLExpr::Function(f) = expr
- && let Some(WindowType::NamedWindow(ident)) = &f.over
- {
- let normalized_ident =
- self.ident_normalizer.normalize(ident.clone());
- for (
- NamedWindowDefinition(_, window_expr),
- normalized_window_ident,
- ) in named_windows.iter()
- {
- if normalized_ident.eq(normalized_window_ident) {
- f.over = Some(match window_expr {
- NamedWindowExpr::NamedWindow(ident) => {
- WindowType::NamedWindow(ident.clone())
- }
- NamedWindowExpr::WindowSpec(spec) => {
- WindowType::WindowSpec(spec.clone())
- }
- })
- }
- }
- // All named windows must be defined with a WindowSpec.
- if let Some(WindowType::NamedWindow(ident)) = &f.over {
- err = Some(plan_err!("The window {ident} is not defined!"));
- return ControlFlow::Break(());
- }
- }
- ControlFlow::Continue(())
- });
- if let Some(err) = err {
- return err;
- }
+ let mut resolver = NamedWindowResolver {
+ planner: self,
+ named_windows: &named_windows,
+ query_depth: 0,
+ };
+ match node.visit(&mut resolver) {
+ ControlFlow::Break(e) => Err(e),
+ ControlFlow::Continue(()) => Ok(()),
+ }
+ }
+
+ /// Returns the window specification of the named window `ident`.
+ ///
+ /// A window definition may only reference windows defined before it, so
+ /// references cannot form a cycle.
+ fn resolve_named_window(
+ &self,
+ ident: &Ident,
+ named_windows: &[(&NamedWindowDefinition, String)],
+ ) -> Result<WindowSpec> {
+ let normalized_ident = self.ident_normalizer.normalize(ident.clone());
+ let Some(position) = named_windows
+ .iter()
+ .position(|(_, name)| *name == normalized_ident)
+ else {
+ return plan_err!("The window {ident} is not defined!");
+ };
+ let (NamedWindowDefinition(_, window_expr), _) = named_windows[position];
+ let earlier_windows = &named_windows[..position];
+ match window_expr {
+ NamedWindowExpr::NamedWindow(base) => {
+ self.resolve_named_window(base, earlier_windows)
+ }
+ NamedWindowExpr::WindowSpec(spec) => {
+ self.resolve_window_spec(spec, earlier_windows)
+ }
+ }
+ }
+
+ /// Resolves a window specification that may reference a named window, as
+ /// in `OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)`.
+ ///
+ /// Following the SQL standard and PostgreSQL, the specification copies
+ /// the referenced window's `PARTITION BY` and `ORDER BY`, may add an
+ /// `ORDER BY` only when the referenced window has none, and may add a
+ /// frame only when the referenced window has none.
+ fn resolve_window_spec(
+ &self,
+ spec: &WindowSpec,
+ named_windows: &[(&NamedWindowDefinition, String)],
+ ) -> Result<WindowSpec> {
+ let Some(base_name) = &spec.window_name else {
+ return Ok(spec.clone());
+ };
+ let base = self.resolve_named_window(base_name, named_windows)?;
+ if !spec.partition_by.is_empty() {
+ return plan_err!(
+ "Cannot override PARTITION BY clause of window {base_name}"
+ );
+ }
+ if !spec.order_by.is_empty() && !base.order_by.is_empty() {
+ return plan_err!("Cannot override ORDER BY clause of window {base_name}");
+ }
+ if base.window_frame.is_some() {
+ return plan_err!(
+ "Cannot copy window {base_name} because it has a frame clause"
+ );
+ }
+ let order_by = if spec.order_by.is_empty() {
+ base.order_by
+ } else {
+ spec.order_by.clone()
+ };
+ Ok(WindowSpec {
+ window_name: None,
+ partition_by: base.partition_by,
+ order_by,
+ window_frame: spec.window_frame.clone(),
+ })
+ }
+}
+
+/// Resolves named window references in the expressions of a single `SELECT`.
+struct NamedWindowResolver<'a, 'b, S: ContextProvider> {
+ planner: &'a SqlToRel<'b, S>,
+ named_windows: &'a [(&'a NamedWindowDefinition, String)],
+ /// Number of subqueries enclosing the expression being visited.
+ query_depth: usize,
+}
+
+impl<S: ContextProvider> VisitorMut for NamedWindowResolver<'_, '_, S> {
+ type Break = DataFusionError;
+
+ fn pre_visit_query(&mut self, _query: &mut Query) -> ControlFlow<Self::Break> {
+ self.query_depth += 1;
+ ControlFlow::Continue(())
+ }
+
+ fn post_visit_query(&mut self, _query: &mut Query) -> ControlFlow<Self::Break> {
+ self.query_depth -= 1;
+ ControlFlow::Continue(())
+ }
+
+ fn post_visit_expr(&mut self, expr: &mut SQLExpr) -> ControlFlow<Self::Break> {
+ if self.query_depth > 0 {
+ return ControlFlow::Continue(());
+ }
+ let SQLExpr::Function(f) = expr else {
+ return ControlFlow::Continue(());
+ };
+ let resolved = match &f.over {
+ Some(WindowType::NamedWindow(ident)) => {
+ self.planner.resolve_named_window(ident, self.named_windows)
+ }
+ Some(WindowType::WindowSpec(spec)) if spec.window_name.is_some() => {
+ self.planner.resolve_window_spec(spec, self.named_windows)
+ }
+ _ => return ControlFlow::Continue(()),
+ };
+ match resolved {
+ Ok(spec) => {
+ f.over = Some(WindowType::WindowSpec(spec));
+ ControlFlow::Continue(())
}
+ Err(e) => ControlFlow::Break(e),
}
- Ok(())
}
}
diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt
index cea8060c13..b224308303 100644
--- a/datafusion/sqllogictest/test_files/window.slt
+++ b/datafusion/sqllogictest/test_files/window.slt
@@ -3481,6 +3481,157 @@ SELECT
ORDER BY C3
LIMIT 5
+# A window that references a named window inherits its PARTITION BY and ORDER BY
+statement ok
+CREATE TABLE named_window_prices(ticker VARCHAR, ts INT, price DOUBLE) AS VALUES
+ ('BTC', 1, 100.0), ('BTC', 2, 105.0), ('BTC', 3, 102.0),
+ ('DOGE', 1, 0.1), ('DOGE', 2, 0.2), ('DOGE', 3, 0.15);
+
+# Adding a frame to a named window keeps its PARTITION BY and ORDER BY
+query TIRRR
+SELECT ticker, ts, price,
+ MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS named_plus_frame,
+ MAX(price) OVER (PARTITION BY ticker ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS inline
+FROM named_window_prices
+WINDOW w AS (PARTITION BY ticker ORDER BY ts)
+ORDER BY ticker, ts
+----
+BTC 1 100 100 100
+BTC 2 105 105 105
+BTC 3 102 105 105
+DOGE 1 0.1 0.1 0.1
+DOGE 2 0.2 0.2 0.2
+DOGE 3 0.15 0.2 0.2
+
+# Adding an ORDER BY to a named window that only partitions
+query TII
+SELECT ticker, ts,
+ ROW_NUMBER() OVER (w ORDER BY ts DESC) AS rn
+FROM named_window_prices
+WINDOW w AS (PARTITION BY ticker)
+ORDER BY ticker, ts
+----
+BTC 1 3
+BTC 2 2
+BTC 3 1
+DOGE 1 3
+DOGE 2 2
+DOGE 3 1
+
+# A window definition can reference an earlier one
+query TIR
+SELECT ticker, ts,
+ SUM(price) OVER w2 AS running
+FROM named_window_prices
+WINDOW w1 AS (PARTITION BY ticker ORDER BY ts),
+ w2 AS (w1 ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
+ORDER BY ticker, ts
+----
+BTC 1 100
+BTC 2 205
+BTC 3 307
+DOGE 1 0.1
+DOGE 2 0.3
+DOGE 3 0.45
+
+# Referencing a named window inside parentheses without adding anything
+query TII
+SELECT ticker, ts, ROW_NUMBER() OVER (w) AS rn
+FROM named_window_prices
+WINDOW w AS (PARTITION BY ticker ORDER BY ts)
+ORDER BY ticker, ts
+----
+BTC 1 1
+BTC 2 2
+BTC 3 3
+DOGE 1 1
+DOGE 2 2
+DOGE 3 3
+
+statement error DataFusion error: Error during planning: The window w is not defined!
+SELECT MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM named_window_prices
+
+statement error DataFusion error: Error during planning: Cannot override PARTITION BY clause of window w
+SELECT MAX(price) OVER (w PARTITION BY ts)
+FROM named_window_prices
+WINDOW w AS (ORDER BY ts)
+
+statement error DataFusion error: Error during planning: Cannot override ORDER BY clause of window w
+SELECT MAX(price) OVER (w ORDER BY price)
+FROM named_window_prices
+WINDOW w AS (PARTITION BY ticker ORDER BY ts)
+
+statement error DataFusion error: Error during planning: Cannot copy window w because it has a frame clause
+SELECT MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
+FROM named_window_prices
+WINDOW w AS (PARTITION BY ticker ORDER BY ts ROWS UNBOUNDED PRECEDING)
+
+# A window definition cannot reference a later one
+statement error DataFusion error: Error during planning: The window w2 is not defined!
+SELECT SUM(price) OVER w1
+FROM named_window_prices
+WINDOW w1 AS (w2 ROWS UNBOUNDED PRECEDING),
+ w2 AS (PARTITION BY ticker ORDER BY ts)
+
+# Named windows are resolved in QUALIFY
+query TIR
+SELECT ticker, ts, price
+FROM named_window_prices
+QUALIFY MAX(price) OVER (w ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) = price
+WINDOW w AS (PARTITION BY ticker ORDER BY ts)
+ORDER BY ticker, ts
+----
+BTC 1 100
+BTC 2 105
+DOGE 1 0.1
+DOGE 2 0.2
+
+query TI
+SELECT ticker, ts
+FROM named_window_prices
+QUALIFY ROW_NUMBER() OVER w = 1
+WINDOW w AS (PARTITION BY ticker ORDER BY ts)
+ORDER BY ticker
+----
+BTC 1
+DOGE 1
+
+# Named windows are resolved in ORDER BY
+query TI
+SELECT ticker, ts
+FROM named_window_prices
+WINDOW w AS (PARTITION BY ticker ORDER BY ts)
+ORDER BY ROW_NUMBER() OVER (w ROWS UNBOUNDED PRECEDING) DESC, ticker
+----
+BTC 3
+DOGE 3
+BTC 2
+DOGE 2
+BTC 1
+DOGE 1
+
+# A subquery resolves window names against its own WINDOW clause
+query I
+SELECT (SELECT MAX(rn) FROM (
+ SELECT ROW_NUMBER() OVER (w ROWS UNBOUNDED PRECEDING) AS rn
+ FROM named_window_prices
+ WINDOW w AS (PARTITION BY ticker ORDER BY ts)))
+FROM named_window_prices
+WINDOW w AS (ORDER BY ts)
+LIMIT 1
+----
+3
+
+statement error DataFusion error: Error during planning: The window w is not defined!
+SELECT (SELECT MAX(rn) FROM (
+ SELECT ROW_NUMBER() OVER w AS rn FROM named_window_prices))
+FROM named_window_prices
+WINDOW w AS (ORDER BY ts)
+
+statement ok
+DROP TABLE named_window_prices;
+
# Create a source where there is multiple orderings.
statement ok
CREATE EXTERNAL TABLE multiple_ordered_table (