Apache DataFusion SQL Query Engine
test: add correlated subquery cases for general decorrelation (#26904)
Mohammad Naqvi74f841f5fdfe
diff --git a/datafusion/sqllogictest/test_files/subquery_decorrelation.slt b/datafusion/sqllogictest/test_files/subquery_decorrelation.sltnew file mode 100644index 0000000000..11e3afddf5--- /dev/null+++ b/datafusion/sqllogictest/test_files/subquery_decorrelation.slt@@ -0,0 +1,1003 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++# https://github.com/apache/datafusion/issues/5492#issuecomment-5934476405+statement ok+CREATE TABLE t1(t1_id INT, t1_name TEXT, t1_int INT) AS VALUES+(11, 'a', 1),+(22, 'b', 2),+(33, 'c', 3),+(44, 'd', 4);++statement ok+CREATE TABLE t2(t2_id INT, t2_name TEXT, t2_int INT) AS VALUES+(11, 'z', 3),+(22, 'y', 1),+(44, 'x', 3),+(55, 'w', 3);++statement ok+CREATE TABLE t3(t3_id INT PRIMARY KEY, t3_name TEXT, t3_int INT) AS VALUES+(11, 'e', 3),+(22, 'f', 1),+(44, 'g', 3),+(55, 'h', 3);++# exists_depth_2_correlation_skipping_a_level+# Fails today. Expected (DuckDB):+# 11+# 22+# 44+query error Schema error: No field named t1\.t1_id+SELECT t1.t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.t2_id = t1.t1_id AND EXISTS (SELECT 1 FROM t3 WHERE t3.t3_id > t1.t1_id));++# exists_depth_2_correlation_one_level_each+query I rowsort+SELECT t1.t1_id FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.t2_id = t1.t1_id AND EXISTS (SELECT 1 FROM t3 WHERE t3.t3_id = t2.t2_id));+----+11+22+44++# exists_in_inner_join_on_reading_both_inputs+# Fails today. Expected (DuckDB):+# 11+# 22+# 44+query error Schema error: No field named t2\.t2_id+SELECT t1.t1_id FROM t1 JOIN t2 ON EXISTS (SELECT 1 FROM t3 WHERE t3.t3_id = t1.t1_id AND t3.t3_id = t2.t2_id);++# exists_in_scalar_subquery_correlated_to_both_levels+# Fails today. Expected (DuckDB):+# 11 1+# 22 1+# 33 0+# 44 1+query error Schema error: No field named t1\.t1_id+SELECT t1.t1_id, (SELECT COUNT(*) FROM t2 WHERE EXISTS (SELECT 1 FROM t3 WHERE t3.t3_id = t2.t2_id AND t3.t3_id = t1.t1_id)) AS c FROM t1;++# outer_column_in_aggregate_argument+# Fails today. Expected (DuckDB):+# 1 14+# 2 18+# 3 22+# 4 26+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.t1_int, (SELECT SUM(t2.t2_int + t1.t1_int) FROM t2) FROM t1;++# outer_column_added_to_aggregate_result+# Fails today. Expected (DuckDB):+# 1 2+# 2 3+# 3 4+# 4 5+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.t1_int, (SELECT MIN(t2.t2_int) + t1.t1_int FROM t2) AS c FROM t1;++# outer_column_projected_under_limit+# Fails today. Expected (DuckDB):+# 1 1+# 2 2+# 3 3+# 4 4+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.t1_int, (SELECT t1.t1_int FROM t2 LIMIT 1) AS c FROM t1;++# greater_than_all_over_expression_with_outer_column+# Fails today. Expected (DuckDB): no rows+query error Physical plan does not support logical expression Exists+SELECT t1.t1_id FROM t1 WHERE t1.t1_id > ALL(SELECT (t2.t2_id + t1.t1_id - 1) / 2 FROM t2 WHERE t2.t2_id IS NOT NULL);++# correlated_inequality_below_count+# Fails today. Expected (DuckDB):+# 1 3+# 2 3+# 3 0+# 4 0+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.t1_int, (SELECT COUNT(*) FROM t2 WHERE t2.t2_int > t1.t1_int) AS c FROM t1;++# outer_column_as_grouping_sets_key+# Fails today. Expected (DuckDB):+# 1 10+# 2 10+# 3 10+# 4 10+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.t1_int, (SELECT SUM(t2.t2_int) FROM t2 GROUP BY GROUPING SETS((t1.t1_int))) AS c FROM t1;++# invalid_aggregate_over_outer_column_without_from+# Fails, as expected, but should fail while planning. Expected (DuckDB): an error, column "t1_int" must appear in the GROUP BY clause+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.t1_int, (SELECT SUM(t1.t1_int)) AS c FROM t1;++# invalid_aggregate_over_outer_column_in_having+# Fails, as expected, but should fail while planning. Expected (DuckDB): an error, WHERE clause cannot contain aggregates+query error Physical plan does not support logical expression Exists+SELECT t1.t1_id FROM t1 WHERE EXISTS (SELECT t2.t2_id FROM t2 GROUP BY t2.t2_id HAVING t2.t2_id > MIN(t1.t1_int));++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++statement ok+DROP TABLE t3;++# https://github.com/apache/datafusion/issues/26710+statement ok+CREATE TABLE invoices (id INT, order_id INT, total INT);++statement ok+INSERT INTO invoices VALUES (1,1,10),(2,1,20),(3,2,30),(4,2,40);++statement ok+CREATE TABLE orders (id INT);++statement ok+INSERT INTO orders VALUES (1),(2),(3);++# correlated_scalar_subquery_order_by_limit_1+# Fails today. Expected (DuckDB):+# 1 20+# 2 40+# 3 NULL+query error Physical plan does not support logical expression ScalarSubquery+SELECT o.id, (SELECT i.total FROM invoices i WHERE i.order_id = o.id ORDER BY i.id DESC LIMIT 1) AS last_total FROM orders o ORDER BY o.id;++statement ok+DROP TABLE invoices;++statement ok+DROP TABLE orders;++# https://github.com/apache/datafusion/issues/16137+statement ok+CREATE TABLE src (id INT, parent_id INT, name VARCHAR);++statement ok+INSERT INTO src VALUES (1, NULL, 'Europe'), (2, 1, 'Warsaw'), (3, 1, 'Paris');++# non_aggregated_correlated_scalar_subquery_over_same_table+# Fails today. Expected (DuckDB):+# 1 Europe NULL+# 2 Warsaw Europe+# 3 Paris Europe+query error Correlated scalar subquery must be aggregated to return at most one row+SELECT id, name, (SELECT p.name FROM src p WHERE p.id = s.parent_id) AS parent_name FROM src s ORDER BY id;++statement ok+DROP TABLE src;++# https://github.com/apache/datafusion/issues/16134+statement ok+CREATE TABLE s (a VARCHAR);++statement ok+INSERT INTO s VALUES ('a'),('b');++# uncorrelated_scalar_subquery_returning_two_rows+# Fails, as expected. Expected (DuckDB): an error, more than one row returned by a subquery+query error Scalar subquery returned more than one row+SELECT s.a, (SELECT x FROM (VALUES (1), (2)) t(x)) sub FROM s ORDER BY 1,2;++statement ok+DROP TABLE s;++statement ok+CREATE TABLE d (k INT, v INT);++statement ok+INSERT INTO d VALUES (1,10),(1,11);++# correlated_scalar_subquery_returning_two_rows_per_outer_row+# Fails, as expected. Expected (DuckDB): an error, more than one row returned by a subquery+query error A GROUP BY clause in a scalar correlated subquery cannot contain non-correlated columns+SELECT k, (SELECT max(v) FROM d d2 WHERE d2.k = d.k GROUP BY d2.k, d2.v) FROM d ORDER BY 1,2;++statement ok+DROP TABLE d;++# https://github.com/apache/datafusion/issues/16134+statement ok+set datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery = false;++statement ok+CREATE TABLE s (a VARCHAR);++statement ok+INSERT INTO s VALUES ('a'),('b');++# uncorrelated_scalar_subquery_returning_two_rows_on_left_join_path+# Wrong today. Expected (DuckDB): an error, more than one row returned by a subquery+query TI+SELECT s.a, (SELECT x FROM (VALUES (1), (2)) t(x)) sub FROM s ORDER BY 1,2;+----+a 1+a 2+b 1+b 2++statement ok+DROP TABLE s;++statement ok+RESET datafusion.optimizer.enable_physical_uncorrelated_scalar_subquery;++# https://github.com/apache/datafusion/issues/15558+statement ok+CREATE TABLE employees (employee_id INT, employee_name VARCHAR, dept_id INT, salary INT);++statement ok+INSERT INTO employees VALUES (1,'a',1,10),(2,'b',1,20),(3,'c',1,30),(4,'d',2,40);++# correlated_scalar_subquery_depth_2+# Fails today. Expected (DuckDB):+# c+# d+query error Schema error: No field named e1\.dept_id+SELECT e1.employee_name FROM employees e1 WHERE e1.salary >= (SELECT AVG(e2.salary) FROM employees e2 WHERE e2.dept_id = e1.dept_id AND e2.salary >= (SELECT AVG(e3.salary) FROM employees e3 WHERE e3.dept_id = e1.dept_id)) ORDER BY 1;++statement ok+DROP TABLE employees;++# https://github.com/apache/datafusion/issues/21727+statement ok+CREATE TABLE t0 (g INT);++statement ok+CREATE TABLE t1 (a INT);++statement ok+CREATE TABLE t2 (c INT);++statement ok+INSERT INTO t0 VALUES (1), (2);++statement ok+INSERT INTO t1 VALUES (10);++statement ok+INSERT INTO t2 VALUES (1);++# exists_grandparent_reference_at_deepest_level+# Fails today. Expected (DuckDB):+# 1+query error Schema error: No field named t0\.g+SELECT g FROM t0 WHERE EXISTS (SELECT 1 FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.c = t0.g)) ORDER BY g;++statement ok+DROP TABLE t0;++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++# https://github.com/apache/datafusion/issues/14554+statement ok+CREATE TABLE t1 (id INT, name VARCHAR, v INT);++statement ok+INSERT INTO t1 VALUES (1,'a',0),(2,'b',5),(3,'z',1);++statement ok+CREATE TABLE t2 (id INT, name VARCHAR);++statement ok+INSERT INTO t2 VALUES (1,'b'),(2,'x'),(3,'x');++# correlated_disjunction_below_count+# Fails today. Expected (DuckDB):+# 1+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.id FROM t1 WHERE (SELECT count(*) FROM t2 WHERE t1.id = t2.id OR t1.name = t2.name) > t1.v ORDER BY 1;++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++statement ok+CREATE TABLE o (k INT);++statement ok+INSERT INTO o VALUES (0),(2),(5);++statement ok+CREATE TABLE i (y INT);++statement ok+INSERT INTO i VALUES (1),(2),(3);++# correlated_range_below_count+# Fails today. Expected (DuckDB):+# 0 3+# 2 1+# 5 0+query error Physical plan does not support logical expression ScalarSubquery+SELECT o.k, (SELECT count(*) FROM i WHERE i.y > o.k) AS c FROM o ORDER BY 1;++statement ok+DROP TABLE o;++statement ok+DROP TABLE i;++# https://github.com/apache/datafusion/issues/25789+statement ok+CREATE TABLE o(k INT) AS VALUES (1), (5);++statement ok+CREATE TABLE a(id INT) AS VALUES (1), (2);++statement ok+CREATE TABLE b(id INT, y INT) AS VALUES (1, 1), (2, 2);++# correlated_filter_below_nullable_side_of_left_join+# Fails today. Expected (DuckDB):+# 1 true+# 5 true+query error Physical plan does not support logical expression Exists+SELECT o.k, EXISTS (SELECT 1 FROM a LEFT JOIN (SELECT * FROM b WHERE b.y = o.k) AS b ON a.id = b.id WHERE b.y IS NULL) AS e FROM o ORDER BY k;++statement ok+DROP TABLE o;++statement ok+DROP TABLE a;++statement ok+DROP TABLE b;++# https://github.com/apache/datafusion/issues/26704+statement ok+CREATE TABLE t1 (a INT, b VARCHAR, x INT);++statement ok+INSERT INTO t1 VALUES (1, 'p', 10), (2, 'p', 20), (3, 'q', 30), (NULL, 'q', 40), (2, 'q', 50);++statement ok+CREATE TABLE t2 (a INT, y INT);++statement ok+INSERT INTO t2 VALUES (2, 7), (2, 9), (3, 5), (NULL, 1);++# exists_in_aggregate_filter+# Fails today. Expected (DuckDB):+# 3+query error Physical plan does not support logical expression Exists+SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.a = t1.a)) AS n FROM t1;++# correlated_scalar_subquery_in_aggregate_argument+# Fails today. Expected (DuckDB):+# p 9+# q 14+query error Physical plan does not support logical expression ScalarSubquery+SELECT t1.b, sum((SELECT max(t2.y) FROM t2 WHERE t2.a = t1.a)) AS s FROM t1 GROUP BY t1.b ORDER BY t1.b;++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++# https://github.com/apache/datafusion/issues/26234+statement ok+CREATE TABLE test(k INT) AS VALUES (1), (2);++# exists_same_named_columns_of_two_inner_relations+# Wrong today. Expected (DuckDB):+# 1+query I+SELECT outer_t.k FROM test outer_t WHERE EXISTS (SELECT 1 FROM test one, test two WHERE one.k = outer_t.k AND two.k = outer_t.k + 1) ORDER BY 1;+----++statement ok+DROP TABLE test;++# https://github.com/apache/datafusion/issues/26014+statement ok+CREATE TABLE o(d DATE) AS VALUES (DATE '2024-01-01'), (DATE '2024-01-02');++statement ok+CREATE TABLE e(ts TIMESTAMP, v INT) AS VALUES (TIMESTAMP '2024-01-01 10:00:00', 1), (TIMESTAMP '2024-01-01 11:00:00', 2), (TIMESTAMP '2024-01-02 09:00:00', 3);++# correlated_lossy_cast_below_count+# Wrong today. Expected (DuckDB):+# 2024-01-01 2+# 2024-01-02 1+query DI+SELECT o.d, (SELECT count(*) FROM e WHERE CAST(e.ts AS DATE) = o.d) AS c FROM o ORDER BY o.d, c;+----+2024-01-01 1+2024-01-01 1+2024-01-02 1++statement ok+DROP TABLE o;++statement ok+DROP TABLE e;++# https://github.com/apache/datafusion/issues/26032+statement ok+CREATE TABLE o(k INT) AS VALUES (1), (2), (5);++statement ok+CREATE TABLE i(id INT, y INT) AS VALUES (1, 1), (2, 2);++# in_with_window_and_repeated_in_predicate+# Wrong today. Expected (DuckDB):+# 1+# 2+query I+SELECT o.k FROM o WHERE o.k IN (SELECT i.y FROM i WHERE i.y = o.k QUALIFY row_number() OVER (ORDER BY i.id) = 1) ORDER BY o.k;+----+1++statement ok+DROP TABLE o;++statement ok+DROP TABLE i;++# https://github.com/apache/datafusion/issues/25808+statement ok+CREATE TABLE po(k INT) AS VALUES (0), (5);++statement ok+CREATE TABLE pb(x INT, y INT) AS VALUES (1, 1), (1, 2), (2, 3);++# lateral_distinct_with_correlated_inequality+# Wrong today. Expected (DuckDB):+# 0 1+# 0 2+query II+SELECT po.k, s.x FROM po, LATERAL (SELECT DISTINCT pb.x FROM pb WHERE pb.y > po.k) AS s ORDER BY po.k, s.x;+----+0 1+0 1+0 2++statement ok+DROP TABLE po;++statement ok+DROP TABLE pb;++# https://github.com/apache/datafusion/issues/24960+statement ok+CREATE TABLE t1(a INT);++statement ok+INSERT INTO t1 VALUES (1),(2);++statement ok+CREATE TABLE t2(a INT, b INT);++statement ok+INSERT INTO t2 VALUES (1,10),(1,NULL);++# in_over_correlated_groupless_count+# Fails today. Expected (DuckDB):+# 2+query error Physical plan does not support logical expression InSubquery+SELECT a FROM t1 WHERE 0 IN (SELECT count(*) FROM t2 WHERE t2.a = t1.a) ORDER BY a;++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++# https://github.com/apache/datafusion/issues/26503+# correlated_count_with_having_on_outer_column+# Fails today. Expected (DuckDB):+# 1 1 1+# 2 1 NULL+# 1 9 NULL+query error Unsupported correlated scalar subquery: a HAVING or filter above a count-like aggregate reads outer columns+SELECT x.a, x.d, (SELECT count(*) FROM (VALUES (1)) AS y(c) WHERE y.c = x.d HAVING count(*) >= x.a) AS v FROM (VALUES (1, 1), (2, 1), (1, 9)) AS x(a, d) ORDER BY x.d, x.a;++# https://github.com/apache/datafusion/issues/26505+# correlated_sum_with_having_true_for_empty_input+# Wrong today. Expected (DuckDB):+# 1 NULL+# 2 NULL+query II+SELECT x.a, (SELECT sum(y.b) FROM (VALUES (1, 10)) AS y(a, b) WHERE y.a = x.a HAVING sum(y.b) IS NULL) AS v FROM (VALUES (1), (2)) AS x(a) ORDER BY x.a;+----+1 10+2 NULL++# https://github.com/apache/datafusion/issues/26508+# aggregate_over_correlated_groupless_aggregate+# Fails today. Expected (DuckDB):+# 1 1+# 2 1+query error Physical plan does not support logical expression ScalarSubquery+SELECT x.a, (SELECT count(*) FROM (SELECT max(y.b) FROM (VALUES (1, 10)) AS y(a, b) WHERE y.a = x.a)) AS v1 FROM (VALUES (1), (2)) AS x(a) ORDER BY x.a;++# https://github.com/apache/datafusion/issues/26695+statement ok+CREATE TABLE t1(a INT) AS VALUES (1), (2), (-1);++statement ok+CREATE TABLE t2(a INT) AS VALUES (1);++# exists_over_groupless_count_with_outer_only_having+# Wrong today. Expected (DuckDB):+# 1+# 2+query I+SELECT a FROM t1 WHERE EXISTS (SELECT count(*) FROM t2 WHERE t2.a = t1.a HAVING t1.a > 0) ORDER BY a;+----+-1+1+2++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++# https://github.com/apache/datafusion/issues/26629+statement ok+CREATE TABLE a(k INT, x INT) AS VALUES (1, 10), (2, 20);++statement ok+CREATE TABLE b(k INT, v INT) AS VALUES (1, 100), (1, 200), (2, 300);++# exists_with_limit_below_having+# Fails today. Expected (DuckDB): no rows+query error Physical plan does not support logical expression Exists+SELECT a.k FROM a WHERE EXISTS (SELECT 1 FROM (SELECT b.k, b.v FROM b WHERE b.k = a.k LIMIT 1) sub HAVING COUNT(*) > 1) ORDER BY 1;++statement ok+DROP TABLE a;++statement ok+DROP TABLE b;++# https://github.com/apache/datafusion/issues/25347+statement ok+CREATE TABLE t_out(id INT, g INT) AS VALUES (1, 1), (2, 2);++statement ok+CREATE TABLE t_in(id INT, g INT) AS VALUES (1, 1), (NULL, 2);++# correlated_not_in_with_null_in_subquery+query I+SELECT id FROM t_out WHERE 3 NOT IN (SELECT id FROM t_in WHERE t_in.g = t_out.g) ORDER BY id;+----+1++statement ok+DROP TABLE t_out;++statement ok+DROP TABLE t_in;++# https://github.com/apache/datafusion/issues/25480+statement ok+CREATE TABLE t1(id INT, z INT) AS VALUES (1,10), (2,20), (NULL,30), (4,40);++statement ok+CREATE TABLE t2(id INT, z INT) AS VALUES (1,5), (NULL,50), (5,10);++# correlated_not_in_with_equality_on_in_value+query I+SELECT id FROM t1 WHERE id NOT IN (SELECT t2.id FROM t2 WHERE t2.id = t1.id) ORDER BY id;+----+2+4+NULL++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++# https://github.com/apache/datafusion/issues/26285+statement ok+CREATE TABLE a(k INT) AS VALUES (1), (100000);++statement ok+CREATE TABLE b(k BIGINT) AS VALUES (3);++statement ok+CREATE TABLE u(c0 INT) AS VALUES (1), (3), (100000);++# exists_on_union_all_column_with_coerced_type+query I+SELECT s.k FROM (SELECT k FROM a UNION ALL SELECT k FROM b) s WHERE EXISTS (SELECT 1 FROM u WHERE u.c0 = s.k) ORDER BY 1;+----+1+3+100000++statement ok+DROP TABLE a;++statement ok+DROP TABLE b;++statement ok+DROP TABLE u;++# https://github.com/apache/datafusion/issues/21198+statement ok+CREATE TABLE o(k INT) AS VALUES (1), (2);++statement ok+CREATE TABLE i(k INT, v INT) AS VALUES (1, 10), (1, 20), (2, 5);++# lateral_with_having+# Fails today. Expected (DuckDB):+# 1 2+query error Physical plan does not support logical expression OuterReferenceColumn+SELECT o.k, s.c FROM o, LATERAL (SELECT count(*) AS c FROM i WHERE i.k = o.k HAVING count(*) > 1) s ORDER BY 1;++statement ok+DROP TABLE o;++statement ok+DROP TABLE i;++# https://github.com/apache/datafusion/issues/21201+statement ok+CREATE TABLE o(k INT) AS VALUES (1), (2);++statement ok+CREATE TABLE i(k INT, v INT) AS VALUES (1, 10), (2, 20);++# lateral_with_outer_column_in_select_list+# Fails today. Expected (DuckDB):+# 1 11+# 2 22+query error Physical plan does not support logical expression OuterReferenceColumn+SELECT o.k, s.w FROM o, LATERAL (SELECT i.v + o.k AS w FROM i WHERE i.k = o.k) s ORDER BY 1;++statement ok+DROP TABLE o;++statement ok+DROP TABLE i;++# https://github.com/apache/datafusion/issues/26573+statement ok+CREATE TABLE t (a INT) AS VALUES (1), (2);++statement ok+CREATE TABLE s (b INT) AS VALUES (1);++# any_in_join_on_over_aliased_table+# Fails today. Expected (DuckDB):+# 1 1+# 1 2+query error Schema error: No field named x\.a+SELECT * FROM t AS x JOIN t AS y ON x.a = ANY (SELECT b FROM s) ORDER BY 1,2;++statement ok+DROP TABLE t;++statement ok+DROP TABLE s;++# https://github.com/apache/datafusion/issues/26622+statement ok+CREATE TABLE t(x INT) AS VALUES (1), (6);++statement ok+CREATE TABLE s(y INT) AS VALUES (5);++# greater_than_all_in_select_list_without_alias+query B rowsort+SELECT x > ALL (SELECT y FROM s) FROM t;+----+false+true++statement ok+DROP TABLE t;++statement ok+DROP TABLE s;++# https://github.com/apache/datafusion/issues/26033+statement ok+CREATE TABLE o(k INT, b BIGINT) AS VALUES (1, 1), (2, 2), (3, 30);++statement ok+CREATE TABLE i(y INT) AS VALUES (1), (2), (3);++# correlated_in_with_cast_subquery_column+# Fails today. Expected (DuckDB):+# 1+# 2+query error Schema error: No field named __correlated_sq_\d+\.y+SELECT o.k FROM o WHERE o.b IN (SELECT i.y FROM i WHERE i.y = o.k) ORDER BY o.k;++statement ok+DROP TABLE o;++statement ok+DROP TABLE i;++# https://github.com/apache/datafusion/issues/11773+# exists_over_correlated_unnest+# Fails today. Expected (DuckDB):+# 2+query error Physical plan does not support logical expression OuterReferenceColumn+SELECT id FROM (SELECT 1 id, [1, 2] arr UNION ALL SELECT 2 id, [3] arr) r WHERE EXISTS (SELECT 1 FROM UNNEST(arr) AS u(x) WHERE u.x > 2) ORDER BY id;++statement ok+CREATE TABLE p(id INT) AS VALUES (1), (2);++statement ok+CREATE TABLE q(pid INT, v INT) AS VALUES (1, 5), (2, 7);++statement ok+CREATE TABLE r(pid INT, v INT) AS VALUES (1, 5), (2, 1);++# correlated_scalar_subquery_in_join_on+# Fails today. Expected (DuckDB):+# 1+query error Correlated scalar subquery can only be used in Projection, Filter, Aggregate and MERGE DML plan nodes+SELECT p.id FROM p JOIN r ON r.pid = p.id AND r.v = (SELECT max(q.v) FROM q WHERE q.pid = p.id) ORDER BY 1;++statement ok+DROP TABLE p;++statement ok+DROP TABLE q;++statement ok+DROP TABLE r;++# https://github.com/apache/datafusion/issues/25837+statement ok+CREATE TABLE o(k INT) AS VALUES (1), (2);++statement ok+CREATE TABLE l(id INT, v INT) AS VALUES (1, 10), (2, 20);++# lateral_with_derived_table_alias+# Fails today. Expected (DuckDB):+# 1 10+# 2 20+query error Schema error: No field named l\.id+SELECT o.k, s.v FROM o, LATERAL (SELECT l2.v FROM (SELECT * FROM l WHERE l.id = o.k) AS l2) AS s ORDER BY o.k;++statement ok+DROP TABLE o;++statement ok+DROP TABLE l;++# https://github.com/apache/datafusion/issues/4656+statement ok+CREATE TABLE t1(id INT, name VARCHAR) AS VALUES (1,'aa'), (2,'bb');++statement ok+CREATE TABLE t2(id INT, name VARCHAR) AS VALUES (1,'aa'), (2,'aa');++# row_valued_in_subquery+query IT+SELECT id, name FROM t2 WHERE (id, name) IN (SELECT id, name FROM t1 WHERE name = 'aa') ORDER BY 1;+----+1 aa++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++# https://github.com/apache/datafusion/issues/25024+statement ok+CREATE TABLE outer_t(id VARCHAR NOT NULL) AS VALUES ('z');++statement ok+CREATE TABLE inner_t(id VARCHAR NOT NULL) AS VALUES ('a');++# nullable_expression_not_in_over_not_null_columns+query T rowsort+SELECT id FROM outer_t WHERE (NULLIF(id, 'z') NOT IN (SELECT id FROM inner_t)) IS NULL;+----+z++statement ok+DROP TABLE outer_t;++statement ok+DROP TABLE inner_t;++# https://github.com/apache/datafusion/issues/26692+statement ok+CREATE TABLE t(a INT, b INT) AS VALUES (1,1),(2,2);++statement ok+CREATE TABLE t2(a INT, b INT) AS VALUES (1,1),(3,30);++# exists_with_two_equal_subplans+# Wrong today. Expected (DuckDB):+# 1+# 1+query I+SELECT t.a FROM t, t2 AS s WHERE EXISTS (SELECT 1 FROM (SELECT * FROM t2 AS s WHERE s.a = t.a) x CROSS JOIN t2 AS s) ORDER BY 1;+----+1++statement ok+DROP TABLE t;++statement ok+DROP TABLE t2;++statement ok+CREATE TABLE t1(a INT) AS VALUES (1), (2), (3);++statement ok+CREATE TABLE t2(a INT) AS VALUES (1);++statement ok+CREATE TABLE t3(a INT) AS VALUES (3);++# exists_with_correlated_union_all_branch+# Fails today. Expected (DuckDB):+# 1+# 3+query error Physical plan does not support logical expression Exists+SELECT a FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.a = t1.a UNION ALL SELECT 1 FROM t3 WHERE t3.a = t1.a) ORDER BY a;++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++statement ok+DROP TABLE t3;++statement ok+CREATE TABLE t1(a INT) AS VALUES (1), (2);++statement ok+CREATE TABLE t2(a INT, y INT) AS VALUES (1, 5);++# correlated_exists_in_case+query IT+SELECT a, CASE WHEN EXISTS (SELECT 1 FROM t2 WHERE t2.a = t1.a) THEN 'y' ELSE 'n' END AS e FROM t1 ORDER BY a;+----+1 y+2 n++# correlated_scalar_subquery_in_case_branch+query II+SELECT a, CASE WHEN a > 1 THEN (SELECT max(y) FROM t2 WHERE t2.a = t1.a) ELSE -1 END AS m FROM t1 ORDER BY a;+----+1 -1+2 NULL++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++statement ok+CREATE TABLE t1(a INT) AS VALUES (1), (2);++statement ok+CREATE TABLE t2(a INT, y INT) AS VALUES (1, 5), (2, 3);++# correlated_scalar_subquery_in_order_by+# Fails today. Expected (DuckDB):+# 2+# 1+query error Correlated scalar subquery can only be used in Projection, Filter, Aggregate and MERGE DML plan nodes+SELECT a FROM t1 ORDER BY (SELECT max(y) FROM t2 WHERE t2.a = t1.a);++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++statement ok+CREATE TABLE s(amount INT, cur VARCHAR) AS VALUES (2,'USD'), (3,'EUR');++statement ok+CREATE TABLE e(amount INT, cur VARCHAR) AS VALUES (10,'USD'), (20,'EUR');++# outer_column_times_aggregate_in_scalar_subquery+# Fails today. Expected (DuckDB):+# 20+# 60+query error Physical plan does not support logical expression ScalarSubquery+SELECT (SELECT max(e.amount) * s.amount FROM e WHERE e.cur = s.cur) AS x FROM s ORDER BY 1;++# outer_column_inside_aggregate_with_correlated_filter+# Fails today. Expected (DuckDB):+# 12+# 23+query error Physical plan does not support logical expression ScalarSubquery+SELECT (SELECT max(e.amount + s.amount) FROM e WHERE e.cur = s.cur) AS x FROM s ORDER BY 1;++statement ok+DROP TABLE s;++statement ok+DROP TABLE e;++statement ok+CREATE TABLE t1(a INT) AS VALUES (1), (2);++statement ok+CREATE TABLE t2(a INT) AS VALUES (1), (2);++statement ok+CREATE TABLE t3(a INT) AS VALUES (1);++# exists_with_outer_column_in_inner_join_on+query I+SELECT a FROM t1 WHERE EXISTS (SELECT 1 FROM t2 JOIN t3 ON t2.a = t3.a AND t3.a = t1.a) ORDER BY a;+----+1++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++statement ok+DROP TABLE t3;++# https://github.com/apache/datafusion/issues/16137+statement ok+CREATE TABLE t1(a INT) AS VALUES (1), (2);++statement ok+CREATE TABLE t2(a INT, y INT) AS VALUES (1, 5), (2, 7);++# non_aggregated_correlated_scalar_subquery_over_another_table+# Fails today. Expected (DuckDB):+# 1 5+# 2 7+query error Correlated scalar subquery must be aggregated to return at most one row+SELECT a, (SELECT y FROM t2 WHERE t2.a = t1.a) AS y FROM t1 ORDER BY a;++statement ok+DROP TABLE t1;++statement ok+DROP TABLE t2;++# https://github.com/apache/datafusion/issues/26572+statement ok+set datafusion.execution.batch_size = 1;++statement ok+CREATE TABLE t (a INT) AS VALUES (1), (2);++# order_by_uncorrelated_scalar_subquery_over_right_join_with_limit+query II+SELECT t.a, (SELECT max(a) FROM t) AS m FROM t AS u RIGHT JOIN t ON u.a = t.a ORDER BY t.a, m LIMIT 5;+----+1 2+2 2++statement ok+DROP TABLE t;++statement ok+RESET datafusion.execution.batch_size;