Apache DataFusion SQL Query Engine
feat: decorrelate a window below a correlated equality filter (#26013)
Oleks Va9fc673ccd17
diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rsindex e79c21dfb7..f6f6668400 100644--- a/datafusion/common/src/nested_struct.rs+++ b/datafusion/common/src/nested_struct.rs@@ -903,7 +903,7 @@ fn validate_map_key_data_type(}Ok(())}- _ if is_injective_map_key_cast(source_type, target_type) => Ok(()),+ _ if is_injective_cast(source_type, target_type) => Ok(()),_ => _plan_err!("Cannot safely evolve Map key type from {} to {}",source_type,@@ -912,15 +912,15 @@ fn validate_map_key_data_type(}}-/// Returns true when casting a Map key from `source_type` to `target_type` is-/// known to preserve key identity.+/// Returns true when casting from `source_type` to `target_type` is known to+/// preserve value identity, so distinct source values stay distinct.///-/// Keep this list conservative. Map keys define equality/lookup identity, so-/// only representation widenings that cannot merge distinct source keys belong-/// here. Other Arrow-supported casts, including decimal/timestamp widening and-/// dictionaries, should stay rejected until their identity semantics are-/// deliberately reviewed.-fn is_injective_map_key_cast(source_type: &DataType, target_type: &DataType) -> bool {+/// Keep this list conservative. Map keys and decorrelated equality keys depend+/// on identity, so only representation widenings that cannot merge distinct+/// source values belong here. Other Arrow-supported casts, including+/// decimal/timestamp widening and dictionaries, should stay rejected until+/// their identity semantics are deliberately reviewed.+pub fn is_injective_cast(source_type: &DataType, target_type: &DataType) -> bool {matches!((source_type, target_type),(@@ -2283,14 +2283,14 @@ mod tests {(DataType::UInt16, DataType::Int64),(DataType::UInt32, DataType::Int64),] {- assert!(is_injective_map_key_cast(&source_type, &target_type));+ assert!(is_injective_cast(&source_type, &target_type));}for (source_type, target_type) in [(DataType::UInt8, DataType::Int8),(DataType::UInt16, DataType::Int16),(DataType::UInt32, DataType::Int32),] {- assert!(!is_injective_map_key_cast(&source_type, &target_type));+ assert!(!is_injective_cast(&source_type, &target_type));}}diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex 7dcf8fde93..e6f235da4d 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -23,11 +23,14 @@ use std::sync::Arc;use crate::push_down_filter::lr_is_preserved;use crate::simplify_expressions::ExprSimplifier;+use datafusion_common::nested_struct::is_injective_cast;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter,};+use datafusion_common::utils::has_float_leaf;use datafusion_common::{- Column, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err, plan_err,+ Column, DFSchema, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err,+ plan_err,};use datafusion_expr::expr::{Alias, GroupingSet};use datafusion_expr::logical_plan::{Join, JoinType};@@ -38,7 +41,7 @@ use datafusion_expr::utils::{use datafusion_expr::{Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, Extension,FetchType, LogicalPlan, LogicalPlanBuilder, MaterializedCte, Operator, SkipType,- UserDefinedLogicalNodeCore, expr, lit,+ UserDefinedLogicalNodeCore, Window, expr, lit,};/// This struct rewrite the sub query plan by pull up the correlated@@ -59,6 +62,10 @@ pub struct PullUpCorrelatedExpr {/// Indicates if we encounter any correlated expression that can not be pulled up/// above a aggregation without changing the meaning of the query.can_pull_over_aggregation: bool,+ /// Whether every correlated conjunct seen so far passes+ /// [`is_partitionable_equality`], so a Window above them can partition by their+ /// columns.+ can_extend_partition_by: bool,/// Do we need to handle [the count bug] during the pull up process.////// The "count bug" was described in [Optimization of Nested SQL@@ -117,6 +124,7 @@ impl PullUpCorrelatedExpr {exists_sub_query: false,can_pull_up: true,can_pull_over_aggregation: true,+ can_extend_partition_by: true,need_handle_count_bug: false,collected_count_expr_map: HashMap::new(),pull_up_having_expr: None,@@ -321,7 +329,9 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {// outer row. A non-equality filter can return duplicates, see// https://github.com/apache/datafusion/issues/25808.// - A Window keeps or drops whole partitions when the filter reads- // only its `PARTITION BY` columns. `f_up` checks this.+ // only its `PARTITION BY` columns. For other filters, `f_up` adds+ // the columns of a column equality to `PARTITION BY` and keeps+ // the subquery correlated in the remaining cases.// - An Unnest maps each input row to its own output rows, so a// filter on a column it passes through keeps the same rows.// The arm above checks that the filter does not read an unnested@@ -384,11 +394,11 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {match &plan {LogicalPlan::Filter(plan_filter) => {let subquery_filter_exprs = split_conjunction(&plan_filter.predicate);- self.can_pull_over_aggregation = self.can_pull_over_aggregation- && subquery_filter_exprs- .iter()- .filter(|e| e.contains_outer())- .all(|&e| can_pullup_over_aggregation(e));+ for &expr in subquery_filter_exprs.iter().filter(|e| e.contains_outer()) {+ self.can_pull_over_aggregation &= can_pullup_over_aggregation(expr);+ self.can_extend_partition_by &=+ is_partitionable_equality(expr, plan_filter.input.schema());+ }for expr in &subquery_filter_exprs {if expr.contains_outer() && !self.correlated_filters.contains(expr) {self.correlated_filters.push((*expr).clone());@@ -713,6 +723,16 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {.iter().all(|expr| partitions_by_all(expr, &local_correlated_cols)){+ // `inner.y = outer.k` keeps the rows of one `y` value for+ // each outer row, so partitioning by `y` as well gives each+ // window function the same rows. An aggregate without+ // `GROUP BY` below returns a row for an outer row without+ // a match, which the pulled up filter would drop.+ if self.can_extend_partition_by && !self.pulled_up_scalar_agg {+ let extended =+ partition_window_by(window, &local_correlated_cols)?;+ return self.pass_through_pulled_up_cols(extended);+ }self.can_pull_up = false;}self.pass_through_pulled_up_cols(plan)@@ -960,6 +980,46 @@ fn correlated_filter_columns(plan: &LogicalPlan) -> Vec<Column> {cols}+/// Adds `cols` to the `PARTITION BY` of every window function of `window`. Each+/// window expression keeps its old output name, so nothing above sees a change.+fn partition_window_by(window: &Window, cols: &BTreeSet<Column>) -> Result<LogicalPlan> {+ let window_expr = window+ .window_expr+ .iter()+ .map(|expr| {+ add_partition_by(expr, cols).alias_if_changed(expr.schema_name().to_string())+ })+ .collect::<Result<Vec<_>>>()?;+ Window::try_new(window_expr, Arc::clone(&window.input)).map(LogicalPlan::Window)+}++fn add_partition_by(expr: &Expr, cols: &BTreeSet<Column>) -> Expr {+ match expr {+ Expr::Alias(alias) => Expr::Alias(Alias {+ expr: Box::new(add_partition_by(&alias.expr, cols)),+ ..alias.clone()+ }),+ Expr::WindowFunction(window_fun) => {+ let mut window_fun = window_fun.clone();+ for col in cols {+ let partitioned = window_fun+ .params+ .partition_by+ .iter()+ .any(|expr| matches!(expr, Expr::Column(c) if c == col));+ if !partitioned {+ window_fun+ .params+ .partition_by+ .push(Expr::Column(col.clone()));+ }+ }+ Expr::WindowFunction(window_fun)+ }+ _ => expr.clone(),+ }+}+/// Whether the window function `expr` lists each of `cols` as a plain/// `PARTITION BY` column.fn partitions_by_all(expr: &Expr, cols: &BTreeSet<Column>) -> bool {@@ -989,30 +1049,54 @@ fn collides_with_existing(exprs: &[Expr], candidate: &Expr) -> bool {}fn can_pullup_over_aggregation(expr: &Expr) -> bool {- if let Expr::BinaryExpr(BinaryExpr {+ split_column_equality(expr).is_some()+}++/// Whether partitioning by the columns of `expr` keeps the rows `expr` matches+/// for one outer row in one partition. That holds for `col = <expr>` or+/// `<expr> = col`, where:+/// - `<expr>` reads no subquery column and is not volatile.+/// - `col` has no float leaf, because `-0.0 = 0.0` holds but a partition+/// keeps the two apart.+/// - `col` is cast only by a cast that keeps distinct values distinct. A lossy+/// cast lets several `col` values match one outer row, see #26014.+fn is_partitionable_equality(expr: &Expr, schema: &DFSchema) -> bool {+ let Some((col_side, other)) = split_column_equality(expr) else {+ return false;+ };+ let (col, cast_to) = match col_side {+ Expr::Cast(Cast { expr, field }) => (&**expr, Some(field.data_type())),+ _ => (col_side, None),+ };+ let Ok(col_type) = col.get_type(schema) else {+ return false;+ };+ !other.is_volatile()+ && !has_float_leaf(&col_type)+ && cast_to.is_none_or(|to| is_injective_cast(&col_type, to))+}++/// Splits `col = <expr>` or `<expr> = col` into the subquery column side, a+/// column or a cast of one, and the other side, which reads no subquery column.+fn split_column_equality(expr: &Expr) -> Option<(&Expr, &Expr)> {+ let Expr::BinaryExpr(BinaryExpr {left,op: Operator::Eq,right,}) = expr- {- match (&**left, &**right) {- (Expr::Column(_), right) => !right.any_column_refs(),- (left, Expr::Column(_)) => !left.any_column_refs(),- (Expr::Cast(Cast { expr, .. }), right)- if matches!(&**expr, Expr::Column(_)) =>- {- !right.any_column_refs()- }- (left, Expr::Cast(Cast { expr, .. }))- if matches!(&**expr, Expr::Column(_)) =>- {- !left.any_column_refs()- }- (_, _) => false,- }- } else {- false- }+ else {+ return None;+ };+ [(&**left, &**right), (&**right, &**left)]+ .into_iter()+ .find(|(col_side, other)| {+ let is_col = match col_side {+ Expr::Column(_) => true,+ Expr::Cast(Cast { expr, .. }) => matches!(&**expr, Expr::Column(_)),+ _ => false,+ };+ is_col && !other.any_column_refs()+ })}fn collect_local_correlated_cols(@@ -1121,7 +1205,11 @@ fn proj_exprs_evaluation_result_on_empty_batch(}).data()?;- if result_expr.ne(expr) {+ // An expression that reads no column, like `1 AS x`, keeps its value+ // on an empty input, while the join that replaces the subquery gives+ // it NULL for an outer row without a match.+ let is_constant = !expr.any_column_refs() && !expr.contains_outer();+ if result_expr.ne(expr) || is_constant {let info = SimplifyContext::builder().with_schema(Arc::clone(schema)).build();diff --git a/datafusion/sqllogictest/test_files/lateral_join.slt b/datafusion/sqllogictest/test_files/lateral_join.sltindex 0e0ceb9cfa..40fb9d7d55 100644--- a/datafusion/sqllogictest/test_files/lateral_join.slt+++ b/datafusion/sqllogictest/test_files/lateral_join.slt@@ -755,16 +755,19 @@ ORDER BY t1.id, sub.x;# --- Correlated filter below a window function ---# https://github.com/apache/datafusion/issues/25792-# Pulling `t2.t1_id = t1.id` above the window would number the t2 rows of all-# t1 rows together instead of the t2 rows of each t1 row.-# Expected result once supported: (1, 10, 1), (1, 20, 2), (2, 30, 1).-statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn+# The pull up adds `t2.t1_id` to the `PARTITION BY` of the window, so each t1+# row still numbers only its own t2 rows.+query IIISELECT t1.id, sub.id, sub.rnFROM t1, LATERAL (SELECT t2.id, row_number() OVER (ORDER BY t2.id) AS rnFROM (SELECT * FROM t2 WHERE t2.t1_id = t1.id) AS t2) AS subORDER BY t1.id, sub.id;+----+1 10 1+1 20 2+2 30 1# A correlated filter above the window is still decorrelated. The window# numbers all t2 rows, then the filter keeps the rows of each t1 row.@@ -850,8 +853,8 @@ ORDER BY t1.id, sub.id;1 20 22 30 1-# The filter must read a `PARTITION BY` column of every window function.-statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn+# Every window function is partitioned by the columns of the filter.+query IIIISELECT t1.id, sub.id, sub.rn, sub.cFROM t1, LATERAL (SELECT t2.id,@@ -860,6 +863,10 @@ FROM t1, LATERAL (FROM t2 WHERE t2.t1_id = t1.id) AS subORDER BY t1.id, sub.id;+----+1 10 1 1+1 20 2 1+2 30 1 1# --- Implicit LATERAL (comma without LATERAL keyword) ---# DuckDB auto-detects lateral correlation; DataFusion requiresdiff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 3cd3bfbf19..a4474c4b03 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -3718,10 +3718,9 @@ DROP TABLE oj_a;statement okDROP TABLE oj_b;-# Regression test for #25792: a correlated filter below a window function must-# not be pulled above the window. The filter selects the rows the window-# function sees for each outer row, so once it is above the window the window-# function runs over all rows and computes different values.+# Regression test for #25792: a correlated filter below a window function+# selects the rows that the window function sees for each outer row. For+# `y = k`, the pull up partitions the window by `y` as well.statement okCREATE TABLE win_outer(k INT) AS VALUES (1), (2), (5);@@ -3731,39 +3730,49 @@ CREATE TABLE win_inner(id INT, y INT) AS VALUES (1, 1), (2, 2);statement okset datafusion.explain.logical_plan_only = true;-# The correlated filter is below the window, so the subquery stays correlated.+# The correlated filter is below the window, so the window is partitioned by b.y.query TTEXPLAIN SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1);----logical_plan-01)Filter: EXISTS (<subquery>)-02)--Subquery:-03)----Projection: Int64(1)-04)------SubqueryAlias: w-05)--------Projection: b.id, row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rn-06)----------Filter: row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW = UInt64(1)-07)------------WindowAggr: windowExpr=[[row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]-08)--------------SubqueryAlias: b-09)----------------Projection: win_inner.id, win_inner.y-10)------------------Filter: win_inner.y = outer_ref(win_outer.k)-11)--------------------TableScan: win_inner-12)--TableScan: win_outer projection=[k]--# It errors rather than returning wrong results. The right answer is `1, 2`.-statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+01)LeftSemi Join: win_outer.k = __correlated_sq_1.y+02)--TableScan: win_outer projection=[k]+03)--SubqueryAlias: __correlated_sq_1+04)----SubqueryAlias: w+05)------Projection: b.y+06)--------Filter: row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW = UInt64(1)+07)----------Projection: b.y, row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW+08)------------WindowAggr: windowExpr=[[row_number() PARTITION BY [b.y] ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]+09)--------------SubqueryAlias: b+10)----------------TableScan: win_inner projection=[id, y]++# The right answer is `1, 2`.+query ISELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1) ORDER BY win_outer.k;+----+1+2# The same for NOT EXISTS. The right answer is `5`.-statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+query ISELECT win_outer.k FROM win_outer WHERE NOT EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1) ORDER BY win_outer.k;+----+5# The same for IN. The right answer is `1, 2`.-statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery+query ISELECT win_outer.k FROM win_outer WHERE 1 IN (SELECT row_number() OVER (ORDER BY win_inner.id) FROM win_inner WHERE win_inner.y = win_outer.k) ORDER BY win_outer.k;+----+1+2# And for a scalar subquery. The right answer is `(1, 1), (2, 1), (5, NULL)`.-statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+query IISELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k;+----+1 1+2 1+5 NULL# A correlated filter above the window is still pulled up, because the window# input does not depend on the outer row.@@ -3808,14 +3817,99 @@ SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (PARTITION2 15 NULL-# A window that partitions by an expression of the column does not count.-statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+# A window that partitions by an expression of the column is partitioned by+# the column as well.+query IISELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (PARTITION BY win_inner.y + 0 ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k;+----+1 1+2 1+5 NULL++# A window below another window whose output has the name the extended window+# would get. The extended window keeps its old name, so the names stay unique.+query II+SELECT win_outer.k, (SELECT max(w.a + w.b) FROM (SELECT row_number() OVER (PARTITION BY win_inner.y ORDER BY win_inner.id) AS a, row_number() OVER (ORDER BY win_inner.id) AS b FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k;+----+1 2+2 2+5 NULL++# A lossless widening cast, as coercion adds for `INT = BIGINT`, keeps the+# distinct values of `y` apart, so the window is partitioned by `y`.+query II+SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = CAST(win_outer.k AS BIGINT)) AS w) AS m FROM win_outer ORDER BY win_outer.k;+----+1 1+2 1+5 NULL++# `-0.0 = 0.0` holds but a window partition keeps the two apart, so a float+# `y` keeps the subquery correlated. The right answer is `(1, 2), (2, NULL), (5, NULL)`.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY v.id) AS rn FROM (VALUES (1, 0.0), (2, -0.0)) AS v(id, f) WHERE v.f = CAST(win_outer.k - 1 AS DOUBLE)) AS w) AS m FROM win_outer ORDER BY win_outer.k;++# A volatile outer expression can match several `y` values for one outer row, so+# the subquery stays correlated.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k + CAST(random() * 0 AS INT)) AS w) AS m FROM win_outer ORDER BY win_outer.k;++# A lossy `CAST(f AS INT) = k` matches several `f` values for one outer row, so+# the subquery stays correlated. The right answer is `(1, 2), (2, NULL), (5, NULL)`.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY v.id) AS rn FROM (VALUES (1, 1.2), (2, 1.4)) AS v(id, f) WHERE CAST(v.f AS INT) = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k;# A window over an aggregate with count bug handling stays correlated.statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquerySELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS m FROM win_outer ORDER BY win_outer.k;+# A projection that drops the aggregate column keeps the count bug handling, so+# the subquery stays correlated. The right answer is `(1, 1), (2, 1), (5, 1)`.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT 1 AS x FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k;++# Without the window, the constant keeps its value for an outer row without a match.+query II+SELECT win_outer.k, (SELECT b.x FROM (SELECT 1 AS x FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k;+----+1 1+2 1+5 1++# An aggregate without `GROUP BY` returns a row for an outer row without a match.+# A window above it keeps the subquery correlated, whatever node sits between+# them. The right answer for each scalar case is `(1, 1), (2, 1), (5, 1)`.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT DISTINCT a.c FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k;++# A Filter that cannot be evaluated on an empty input.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery+SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT a.c FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a WHERE random() >= 0) AS b) AS m FROM win_outer ORDER BY win_outer.k;++# The same for EXISTS, NOT EXISTS and IN, which do not track count bug handling.+# The right answers are `1, 2, 5`, no rows, and `1, 2, 5`.+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k;++statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists+SELECT win_outer.k FROM win_outer WHERE NOT EXISTS (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k;++statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery+SELECT win_outer.k FROM win_outer WHERE 1 IN (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k;++# And for LATERAL.+statement error DataFusion error: This feature is not implemented: Physical plan does not support undecorrelated Subquery+SELECT win_outer.k, s.n FROM win_outer, LATERAL (SELECT count(*) OVER () AS n FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS s ORDER BY win_outer.k;++# An aggregate with `GROUP BY` returns no row for an outer row without a match,+# so the window above it is partitioned.+query II+SELECT win_outer.k, (SELECT max(w.n) FROM (SELECT count(*) OVER () AS n FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k GROUP BY win_inner.y) AS a) AS w) AS m FROM win_outer ORDER BY win_outer.k;+----+1 1+2 1+5 NULL+# LEFT JOIN LATERAL keeps the outer rows without a match.query IIISELECT win_outer.k, s.id, s.rn FROM win_outer LEFT JOIN LATERAL (SELECT win_inner.id, row_number() OVER (PARTITION BY win_inner.y ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS s ON true ORDER BY win_outer.k;