Skip to content

Apache DataFusion SQL Query Engine

Add more tests to projection_pushdown.slt (#20236)

Adrian Garcia Badaraccoaebd100e137e
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index dd8ca26e4c..a8c985f963 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -1278,6 +1278,7 @@ SELECT id, s['value'] FROM simple_struct ORDER BY id, s['label'];
4 300
5 250
+
###
# Test 11a.3: TopK with dropped sort column
# Same as test 11a.1 but with LIMIT
@@ -1608,3 +1609,188 @@ physical_plan
02)--FilterExec: id@0 > 2, projection=[s@1]
03)----RepartitionExec: partitioning=RoundRobinBatch(32), input_partitions=1
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]
+
+#####################
+# Section 14: SubqueryAlias tests
+#####################
+
+# Reset target partitions
+statement ok
+SET datafusion.execution.target_partitions = 1;
+
+# get_field pushdown through subquery alias with filter
+query TT
+EXPLAIN SELECT t.s['value'] FROM (SELECT * FROM simple_struct) t WHERE t.id > 2;
+----
+logical_plan
+01)Projection: get_field(t.s, Utf8("value"))
+02)--SubqueryAlias: t
+03)----Projection: simple_struct.s
+04)------Filter: simple_struct.id > Int64(2)
+05)--------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(2)]
+physical_plan
+01)ProjectionExec: expr=[get_field(s@0, value) as t.s[value]]
+02)--FilterExec: id@0 > 2, projection=[s@1]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]
+
+# Verify correctness
+query I
+SELECT t.s['value'] FROM (SELECT * FROM simple_struct) t WHERE t.id > 2 ORDER BY t.id;
+----
+150
+300
+250
+
+# Multiple get_field through subquery alias with sort
+query TT
+EXPLAIN SELECT t.s['value'], t.s['label'] FROM (SELECT * FROM simple_struct) t ORDER BY t.s['value'];
+----
+logical_plan
+01)Sort: t.s[value] ASC NULLS LAST
+02)--Projection: get_field(t.s, Utf8("value")), get_field(t.s, Utf8("label"))
+03)----SubqueryAlias: t
+04)------TableScan: simple_struct projection=[s]
+physical_plan
+01)SortExec: expr=[t.s[value]@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as t.s[value], get_field(s@1, label) as t.s[label]], file_type=parquet
+
+# Verify correctness
+query IT
+SELECT t.s['value'], t.s['label'] FROM (SELECT * FROM simple_struct) t ORDER BY t.s['value'];
+----
+100 alpha
+150 gamma
+200 beta
+250 epsilon
+300 delta
+
+# Nested subquery aliases
+query TT
+EXPLAIN SELECT u.s['value'] FROM (SELECT * FROM (SELECT * FROM simple_struct) t) u WHERE u.id > 2;
+----
+logical_plan
+01)Projection: get_field(u.s, Utf8("value"))
+02)--SubqueryAlias: u
+03)----SubqueryAlias: t
+04)------Projection: simple_struct.s
+05)--------Filter: simple_struct.id > Int64(2)
+06)----------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(2)]
+physical_plan
+01)ProjectionExec: expr=[get_field(s@0, value) as u.s[value]]
+02)--FilterExec: id@0 > 2, projection=[s@1]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]
+
+# Verify correctness
+query I
+SELECT u.s['value'] FROM (SELECT * FROM (SELECT * FROM simple_struct) t) u WHERE u.id > 2 ORDER BY u.id;
+----
+150
+300
+250
+
+# get_field in filter through subquery alias
+query TT
+EXPLAIN SELECT t.id FROM (SELECT * FROM simple_struct) t WHERE t.s['value'] > 200;
+----
+logical_plan
+01)SubqueryAlias: t
+02)--Projection: simple_struct.id
+03)----Filter: get_field(simple_struct.s, Utf8("value")) > Int64(200)
+04)------TableScan: simple_struct projection=[id, s], partial_filters=[get_field(simple_struct.s, Utf8("value")) > Int64(200)]
+physical_plan
+01)FilterExec: get_field(s@1, value) > 200, projection=[id@0]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet
+
+# Verify correctness
+query I
+SELECT t.id FROM (SELECT * FROM simple_struct) t WHERE t.s['value'] > 200 ORDER BY t.id;
+----
+4
+5
+
+#####################
+# Section 15: UNION ALL tests
+#####################
+
+# get_field on UNION ALL result
+query TT
+EXPLAIN SELECT s['value'] FROM (
+ SELECT s FROM simple_struct WHERE id <= 3
+ UNION ALL
+ SELECT s FROM simple_struct WHERE id > 3
+) t;
+----
+logical_plan
+01)Projection: get_field(t.s, Utf8("value"))
+02)--SubqueryAlias: t
+03)----Union
+04)------Projection: simple_struct.s
+05)--------Filter: simple_struct.id <= Int64(3)
+06)----------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id <= Int64(3)]
+07)------Projection: simple_struct.s
+08)--------Filter: simple_struct.id > Int64(3)
+09)----------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(3)]
+physical_plan
+01)ProjectionExec: expr=[get_field(s@0, value) as t.s[value]]
+02)--UnionExec
+03)----FilterExec: id@0 <= 3, projection=[s@1]
+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 <= 3, pruning_predicate=id_null_count@1 != row_count@2 AND id_min@0 <= 3, required_guarantees=[]
+05)----FilterExec: id@0 > 3, projection=[s@1]
+06)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 3, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 3, required_guarantees=[]
+
+# Verify correctness
+query I
+SELECT s['value'] FROM (
+ SELECT s FROM simple_struct WHERE id <= 3
+ UNION ALL
+ SELECT s FROM simple_struct WHERE id > 3
+) t ORDER BY s['value'];
+----
+100
+150
+200
+250
+300
+
+# Multiple get_field on UNION ALL with ORDER BY
+query TT
+EXPLAIN SELECT s['value'], s['label'] FROM (
+ SELECT s FROM simple_struct WHERE id <= 3
+ UNION ALL
+ SELECT s FROM simple_struct WHERE id > 3
+) t ORDER BY s['value'];
+----
+logical_plan
+01)Sort: t.s[value] ASC NULLS LAST
+02)--Projection: get_field(t.s, Utf8("value")), get_field(t.s, Utf8("label"))
+03)----SubqueryAlias: t
+04)------Union
+05)--------Projection: simple_struct.s
+06)----------Filter: simple_struct.id <= Int64(3)
+07)------------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id <= Int64(3)]
+08)--------Projection: simple_struct.s
+09)----------Filter: simple_struct.id > Int64(3)
+10)------------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(3)]
+physical_plan
+01)SortPreservingMergeExec: [t.s[value]@0 ASC NULLS LAST]
+02)--SortExec: expr=[t.s[value]@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----ProjectionExec: expr=[get_field(s@0, value) as t.s[value], get_field(s@0, label) as t.s[label]]
+04)------UnionExec
+05)--------FilterExec: id@0 <= 3, projection=[s@1]
+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 <= 3, pruning_predicate=id_null_count@1 != row_count@2 AND id_min@0 <= 3, required_guarantees=[]
+07)--------FilterExec: id@0 > 3, projection=[s@1]
+08)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, s], file_type=parquet, predicate=id@0 > 3, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 3, required_guarantees=[]
+
+# Verify correctness
+query IT
+SELECT s['value'], s['label'] FROM (
+ SELECT s FROM simple_struct WHERE id <= 3
+ UNION ALL
+ SELECT s FROM simple_struct WHERE id > 3
+) t ORDER BY s['value'];
+----
+100 alpha
+150 gamma
+200 beta
+250 epsilon
+300 delta