Apache DataFusion SQL Query Engine
Support pushing down empty projections into joins (#20191)
Jack Kleemanaf5f47046561
diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rsindex d9b36dc4b8..00e016ae02 100644--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs@@ -1723,3 +1723,47 @@ fn test_cooperative_exec_after_projection() -> Result<()> {Ok(())}++#[test]+fn test_hash_join_empty_projection_embeds() -> Result<()> {+ let left_csv = create_simple_csv_exec();+ let right_csv = create_simple_csv_exec();++ let join = Arc::new(HashJoinExec::try_new(+ left_csv,+ right_csv,+ vec![(Arc::new(Column::new("a", 0)), Arc::new(Column::new("a", 0)))],+ None,+ &JoinType::Right,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ false,+ )?);++ // Empty projection: no columns needed from the join output+ let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(+ vec![] as Vec<ProjectionExpr>,+ join,+ )?);++ let after_optimize =+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;+ let after_optimize_string = displayable(after_optimize.as_ref())+ .indent(true)+ .to_string();+ let actual = after_optimize_string.trim();++ // The empty projection should be embedded into the HashJoinExec,+ // resulting in projection=[] on the join and no ProjectionExec wrapper.+ assert_snapshot!(+ actual,+ @r"+ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@0, a@0)], projection=[]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+ "+ );++ Ok(())+}diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex e709703e07..83fd418d73 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -977,6 +977,17 @@ pub(crate) fn apply_join_filter_to_indices())}+/// Creates a [RecordBatch] with zero columns but the given row count.+/// Used when a join has an empty projection (e.g. `SELECT count(1) ...`).+fn new_empty_schema_batch(schema: &Schema, row_count: usize) -> Result<RecordBatch> {+ let options = RecordBatchOptions::new().with_row_count(Some(row_count));+ Ok(RecordBatch::try_new_with_options(+ Arc::new(schema.clone()),+ vec![],+ &options,+ )?)+}+/// Returns a new [RecordBatch] by combining the `left` and `right` according to `indices`./// The resulting batch has [Schema] `schema`.pub(crate) fn build_batch_from_indices(@@ -989,15 +1000,7 @@ pub(crate) fn build_batch_from_indices(build_side: JoinSide,) -> Result<RecordBatch> {if schema.fields().is_empty() {- let options = RecordBatchOptions::new()- .with_match_field_names(true)- .with_row_count(Some(build_indices.len()));-- return Ok(RecordBatch::try_new_with_options(- Arc::new(schema.clone()),- vec![],- &options,- )?);+ return new_empty_schema_batch(schema, build_indices.len());}// build the columns of the new [RecordBatch]:@@ -1057,6 +1060,9 @@ pub(crate) fn build_batch_empty_build_side(// the remaining joins will return data for the right columns and null for the left onesJoinType::Right | JoinType::Full | JoinType::RightAnti | JoinType::RightMark => {let num_rows = probe_batch.num_rows();+ if schema.fields().is_empty() {+ return new_empty_schema_batch(schema, num_rows);+ }let mut columns: Vec<Arc<dyn Array>> =Vec::with_capacity(schema.fields().len());@@ -2889,4 +2895,35 @@ mod tests {Ok(())}++ #[test]+ fn test_build_batch_empty_build_side_empty_schema() -> Result<()> {+ // When the output schema has no fields (empty projection pushed into+ // the join), build_batch_empty_build_side should return a RecordBatch+ // with the correct row count but no columns.+ let empty_schema = Schema::empty();++ let build_batch = RecordBatch::try_new(+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])),+ vec![Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3]))],+ )?;++ let probe_batch = RecordBatch::try_new(+ Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)])),+ vec![Arc::new(arrow::array::Int32Array::from(vec![4, 5, 6, 7]))],+ )?;++ let result = build_batch_empty_build_side(+ &empty_schema,+ &build_batch,+ &probe_batch,+ &[], // no column indices with empty projection+ JoinType::Right,+ )?;++ assert_eq!(result.num_rows(), 4);+ assert_eq!(result.num_columns(), 0);++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 76711a8f83..8ab87c60d8 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -553,6 +553,15 @@ impl RecordBatchStream for ProjectionStream {}}+/// Trait for execution plans that can embed a projection, avoiding a separate+/// [`ProjectionExec`] wrapper.+///+/// # Empty projections+///+/// `Some(vec![])` is a valid projection that produces zero output columns while+/// preserving the correct row count. Implementors must ensure that runtime batch+/// construction still returns batches with the right number of rows even when no+/// columns are selected (e.g. for `SELECT count(1) … JOIN …`).pub trait EmbeddedProjection: ExecutionPlan + Sized {fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self>;}@@ -563,6 +572,15 @@ pub fn try_embed_projection<Exec: EmbeddedProjection + 'static>(projection: &ProjectionExec,execution_plan: &Exec,) -> Result<Option<Arc<dyn ExecutionPlan>>> {+ // If the projection has no expressions at all (e.g., ProjectionExec: expr=[]),+ // embed an empty projection into the execution plan so it outputs zero columns.+ // This avoids allocating throwaway null arrays for build-side columns+ // when no output columns are actually needed (e.g., count(1) over a right join).+ if projection.expr().is_empty() {+ let new_execution_plan = Arc::new(execution_plan.with_projection(Some(vec![]))?);+ return Ok(Some(new_execution_plan));+ }+// Collect all column indices from the given projection expressions.let projection_index = collect_column_indices(projection.expr());diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.sltindex 8c0752d2ed..a41cbecf77 100644--- a/datafusion/sqllogictest/test_files/array.slt+++ b/datafusion/sqllogictest/test_files/array.slt@@ -6606,10 +6606,9 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]-05)--------ProjectionExec: expr=[]-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]query Iwith test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))@@ -6634,10 +6633,9 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]-05)--------ProjectionExec: expr=[]-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]query Iwith test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))@@ -6662,10 +6660,9 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]-05)--------ProjectionExec: expr=[]-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]query Iwith test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))@@ -6690,10 +6687,9 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]-05)--------ProjectionExec: expr=[]-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]query Iwith test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))@@ -6718,10 +6714,9 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]-05)--------ProjectionExec: expr=[]-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]query Iwith test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))@@ -6748,10 +6743,9 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]-05)--------ProjectionExec: expr=[]-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IS NOT NULL OR NULL-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IS NOT NULL OR NULL, projection=[]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]# any operatorquery ?diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex dd7f4710d9..3edc721afc 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -2085,7 +2085,7 @@ SELECT join_t1.t1_id, join_t2.t2_idFROM (select t1_id from join_t1 where join_t1.t1_id > 22) as join_t1RIGHT JOIN (select t2_id from join_t2 where join_t2.t2_id > 11) as join_t2ON join_t1.t1_id < join_t2.t2_id-ORDER BY 1, 2+ORDER BY 1, 2----33 4433 55@@ -3954,7 +3954,7 @@ query TTexplain select t1_id, t1_name, i from join_t1 t1 cross join lateral (select * from unnest(generate_series(1, t1_int))) as series(i);----logical_plan-01)Cross Join:+01)Cross Join:02)--SubqueryAlias: t103)----TableScan: join_t1 projection=[t1_id, t1_name]04)--SubqueryAlias: series@@ -4367,10 +4367,9 @@ logical_planphysical_plan01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)]02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))]-03)----ProjectionExec: expr=[]-04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(binary_col@0, binary_col@0)]-05)--------DataSourceExec: partitions=1, partition_sizes=[1]-06)--------DataSourceExec: partitions=1, partition_sizes=[1]+03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(binary_col@0, binary_col@0)], projection=[]+04)------DataSourceExec: partitions=1, partition_sizes=[1]+05)------DataSourceExec: partitions=1, partition_sizes=[1]# Test hash join sort push down# Issue: https://github.com/apache/datafusion/issues/13559@@ -4532,7 +4531,7 @@ query TTexplain SELECT * FROM person a NATURAL JOIN lineitem b;----logical_plan-01)Cross Join:+01)Cross Join:02)--SubqueryAlias: a03)----TableScan: person projection=[id, age, state]04)--SubqueryAlias: b@@ -4578,7 +4577,7 @@ query TTexplain SELECT j1_string, j2_string FROM j1, LATERAL (SELECT * FROM j2 WHERE j1_id < j2_id) AS j2;----logical_plan-01)Cross Join:+01)Cross Join:02)--TableScan: j1 projection=[j1_string]03)--SubqueryAlias: j204)----Projection: j2.j2_string@@ -4591,7 +4590,7 @@ query TTexplain SELECT * FROM j1 JOIN (j2 JOIN j3 ON(j2_id = j3_id - 2)) ON(j1_id = j2_id), LATERAL (SELECT * FROM j3 WHERE j3_string = j2_string) as j4----logical_plan-01)Cross Join:+01)Cross Join:02)--Inner Join: CAST(j2.j2_id AS Int64) = CAST(j3.j3_id AS Int64) - Int64(2)03)----Inner Join: j1.j1_id = j2.j2_id04)------TableScan: j1 projection=[j1_string, j1_id]@@ -4607,11 +4606,11 @@ query TTexplain SELECT * FROM j1, LATERAL (SELECT * FROM j1, LATERAL (SELECT * FROM j2 WHERE j1_id = j2_id) as j2) as j2;----logical_plan-01)Cross Join:+01)Cross Join:02)--TableScan: j1 projection=[j1_string, j1_id]03)--SubqueryAlias: j204)----Subquery:-05)------Cross Join:+05)------Cross Join:06)--------TableScan: j1 projection=[j1_string, j1_id]07)--------SubqueryAlias: j208)----------Subquery:@@ -4623,7 +4622,7 @@ query TTexplain SELECT j1_string, j2_string FROM j1 LEFT JOIN LATERAL (SELECT * FROM j2 WHERE j1_id < j2_id) AS j2 ON(true);----logical_plan-01)Left Join:+01)Left Join:02)--TableScan: j1 projection=[j1_string]03)--SubqueryAlias: j204)----Projection: j2.j2_string@@ -4636,9 +4635,9 @@ query TTexplain SELECT * FROM j1, (j2 LEFT JOIN LATERAL (SELECT * FROM j3 WHERE j1_id + j2_id = j3_id) AS j3 ON(true));----logical_plan-01)Cross Join:+01)Cross Join:02)--TableScan: j1 projection=[j1_string, j1_id]-03)--Left Join:+03)--Left Join:04)----TableScan: j2 projection=[j2_string, j2_id]05)----SubqueryAlias: j306)------Subquery:@@ -4650,7 +4649,7 @@ query TTexplain SELECT * FROM j1, LATERAL (SELECT 1) AS j2;----logical_plan-01)Cross Join:+01)Cross Join:02)--TableScan: j1 projection=[j1_string, j1_id]03)--SubqueryAlias: j204)----Projection: Int64(1)@@ -4992,7 +4991,7 @@ FULL JOIN t2 ON k1 = k2# LEFT MARK JOINquery TT-EXPLAIN+EXPLAINSELECT *FROM t2WHERE k2 > 0@@ -5067,14 +5066,14 @@ CREATE OR REPLACE TABLE t1(b INT, c INT, d INT);statement okINSERT INTO t1 VALUES- (10, 5, 3),- ( 1, 7, 8),- ( 2, 9, 7),- ( 3, 8,10),- ( 5, 6, 6),- ( 0, 4, 9),- ( 4, 8, 7),- (100,6, 5);+ (10, 5, 3),+ ( 1, 7, 8),+ ( 2, 9, 7),+ ( 3, 8,10),+ ( 5, 6, 6),+ ( 0, 4, 9),+ ( 4, 8, 7),+ (100,6, 5);query I rowsortSELECT c@@ -5268,3 +5267,30 @@ DROP TABLE issue_19067_left;statement count 0DROP TABLE issue_19067_right;++# Test that empty projections pushed into joins produce correct row counts at runtime.+# When count(1) is used over a RIGHT/FULL JOIN, the optimizer embeds an empty projection+# (projection=[]) into the HashJoinExec. This validates that the runtime batch construction+# handles zero-column output correctly, preserving the correct number of rows.++statement ok+CREATE TABLE empty_proj_left AS VALUES (1, 'a'), (2, 'b'), (3, 'c');++statement ok+CREATE TABLE empty_proj_right AS VALUES (1, 'x'), (2, 'y'), (4, 'z');++query I+SELECT count(1) FROM empty_proj_left RIGHT JOIN empty_proj_right ON empty_proj_left.column1 = empty_proj_right.column1;+----+3++query I+SELECT count(1) FROM empty_proj_left FULL JOIN empty_proj_right ON empty_proj_left.column1 = empty_proj_right.column1;+----+4++statement count 0+DROP TABLE empty_proj_left;++statement count 0+DROP TABLE empty_proj_right;diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.sltindex 5a44112334..e18114bc51 100644--- a/datafusion/sqllogictest/test_files/projection.slt+++ b/datafusion/sqllogictest/test_files/projection.slt@@ -167,12 +167,12 @@ set datafusion.explain.logical_plan_only = false# project cast dictionaryquery T-SELECT- CASE+SELECT+ CASEWHEN cpu_load_short.host IS NULL THEN ''ELSE cpu_load_short.hostEND AS host-FROM+FROMcpu_load_short;----host1@@ -275,7 +275,6 @@ logical_plan02)--Filter: t1.a > Int64(1)03)----TableScan: t1 projection=[a], partial_filters=[t1.a > Int64(1)]physical_plan-01)ProjectionExec: expr=[]-02)--FilterExec: a@0 > 1-03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[]+01)FilterExec: a@0 > 1, projection=[]+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[]