Skip to content

Apache DataFusion SQL Query Engine

Support pushing down empty projections into joins (#20191)

Jack Kleemanaf5f47046561
diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
index d9b36dc4b8..00e016ae02 100644
--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
@@ -1723,3 +1723,47 @@ fn test_cooperative_exec_after_projection() -> Result<()> {
Ok(())
}
+
+#[test]
+fn test_hash_join_empty_projection_embeds() -> Result<()> {
+ let left_csv = create_simple_csv_exec();
+ let right_csv = create_simple_csv_exec();
+
+ let join = Arc::new(HashJoinExec::try_new(
+ left_csv,
+ right_csv,
+ vec![(Arc::new(Column::new("a", 0)), Arc::new(Column::new("a", 0)))],
+ None,
+ &JoinType::Right,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ false,
+ )?);
+
+ // Empty projection: no columns needed from the join output
+ let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(
+ vec![] as Vec<ProjectionExpr>,
+ join,
+ )?);
+
+ let after_optimize =
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
+ let after_optimize_string = displayable(after_optimize.as_ref())
+ .indent(true)
+ .to_string();
+ let actual = after_optimize_string.trim();
+
+ // The empty projection should be embedded into the HashJoinExec,
+ // resulting in projection=[] on the join and no ProjectionExec wrapper.
+ assert_snapshot!(
+ actual,
+ @r"
+ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@0, a@0)], projection=[]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+ "
+ );
+
+ Ok(())
+}
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index e709703e07..83fd418d73 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -977,6 +977,17 @@ pub(crate) fn apply_join_filter_to_indices(
))
}
+/// Creates a [RecordBatch] with zero columns but the given row count.
+/// Used when a join has an empty projection (e.g. `SELECT count(1) ...`).
+fn new_empty_schema_batch(schema: &Schema, row_count: usize) -> Result<RecordBatch> {
+ let options = RecordBatchOptions::new().with_row_count(Some(row_count));
+ Ok(RecordBatch::try_new_with_options(
+ Arc::new(schema.clone()),
+ vec![],
+ &options,
+ )?)
+}
+
/// Returns a new [RecordBatch] by combining the `left` and `right` according to `indices`.
/// The resulting batch has [Schema] `schema`.
pub(crate) fn build_batch_from_indices(
@@ -989,15 +1000,7 @@ pub(crate) fn build_batch_from_indices(
build_side: JoinSide,
) -> Result<RecordBatch> {
if schema.fields().is_empty() {
- let options = RecordBatchOptions::new()
- .with_match_field_names(true)
- .with_row_count(Some(build_indices.len()));
-
- return Ok(RecordBatch::try_new_with_options(
- Arc::new(schema.clone()),
- vec![],
- &options,
- )?);
+ return new_empty_schema_batch(schema, build_indices.len());
}
// build the columns of the new [RecordBatch]:
@@ -1057,6 +1060,9 @@ pub(crate) fn build_batch_empty_build_side(
// the remaining joins will return data for the right columns and null for the left ones
JoinType::Right | JoinType::Full | JoinType::RightAnti | JoinType::RightMark => {
let num_rows = probe_batch.num_rows();
+ if schema.fields().is_empty() {
+ return new_empty_schema_batch(schema, num_rows);
+ }
let mut columns: Vec<Arc<dyn Array>> =
Vec::with_capacity(schema.fields().len());
@@ -2889,4 +2895,35 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_build_batch_empty_build_side_empty_schema() -> Result<()> {
+ // When the output schema has no fields (empty projection pushed into
+ // the join), build_batch_empty_build_side should return a RecordBatch
+ // with the correct row count but no columns.
+ let empty_schema = Schema::empty();
+
+ let build_batch = RecordBatch::try_new(
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])),
+ vec![Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3]))],
+ )?;
+
+ let probe_batch = RecordBatch::try_new(
+ Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)])),
+ vec![Arc::new(arrow::array::Int32Array::from(vec![4, 5, 6, 7]))],
+ )?;
+
+ let result = build_batch_empty_build_side(
+ &empty_schema,
+ &build_batch,
+ &probe_batch,
+ &[], // no column indices with empty projection
+ JoinType::Right,
+ )?;
+
+ assert_eq!(result.num_rows(), 4);
+ assert_eq!(result.num_columns(), 0);
+
+ Ok(())
+ }
}
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index 76711a8f83..8ab87c60d8 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -553,6 +553,15 @@ impl RecordBatchStream for ProjectionStream {
}
}
+/// Trait for execution plans that can embed a projection, avoiding a separate
+/// [`ProjectionExec`] wrapper.
+///
+/// # Empty projections
+///
+/// `Some(vec![])` is a valid projection that produces zero output columns while
+/// preserving the correct row count. Implementors must ensure that runtime batch
+/// construction still returns batches with the right number of rows even when no
+/// columns are selected (e.g. for `SELECT count(1) … JOIN …`).
pub trait EmbeddedProjection: ExecutionPlan + Sized {
fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self>;
}
@@ -563,6 +572,15 @@ pub fn try_embed_projection<Exec: EmbeddedProjection + 'static>(
projection: &ProjectionExec,
execution_plan: &Exec,
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
+ // If the projection has no expressions at all (e.g., ProjectionExec: expr=[]),
+ // embed an empty projection into the execution plan so it outputs zero columns.
+ // This avoids allocating throwaway null arrays for build-side columns
+ // when no output columns are actually needed (e.g., count(1) over a right join).
+ if projection.expr().is_empty() {
+ let new_execution_plan = Arc::new(execution_plan.with_projection(Some(vec![]))?);
+ return Ok(Some(new_execution_plan));
+ }
+
// Collect all column indices from the given projection expressions.
let projection_index = collect_column_indices(projection.expr());
diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt
index 8c0752d2ed..a41cbecf77 100644
--- a/datafusion/sqllogictest/test_files/array.slt
+++ b/datafusion/sqllogictest/test_files/array.slt
@@ -6606,10 +6606,9 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
-05)--------ProjectionExec: expr=[]
-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
query I
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6634,10 +6633,9 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
-05)--------ProjectionExec: expr=[]
-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
query I
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6662,10 +6660,9 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
-05)--------ProjectionExec: expr=[]
-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
query I
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6690,10 +6687,9 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
-05)--------ProjectionExec: expr=[]
-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
query I
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6718,10 +6714,9 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
-05)--------ProjectionExec: expr=[]
-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c])
-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IN (SET) ([7f4b18de3cfeb9b4ac78c381ee2ad278, a, b, c]), projection=[]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
query I
with test AS (SELECT substr(md5(i::text)::text, 1, 32) as needle FROM generate_series(1, 100000) t(i))
@@ -6748,10 +6743,9 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]
-05)--------ProjectionExec: expr=[]
-06)----------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IS NOT NULL OR NULL
-07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-08)--------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
+05)--------FilterExec: substr(md5(CAST(value@0 AS Utf8View)), 1, 32) IS NOT NULL OR NULL, projection=[]
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192]
# any operator
query ?
diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt
index dd7f4710d9..3edc721afc 100644
--- a/datafusion/sqllogictest/test_files/joins.slt
+++ b/datafusion/sqllogictest/test_files/joins.slt
@@ -2085,7 +2085,7 @@ SELECT join_t1.t1_id, join_t2.t2_id
FROM (select t1_id from join_t1 where join_t1.t1_id > 22) as join_t1
RIGHT JOIN (select t2_id from join_t2 where join_t2.t2_id > 11) as join_t2
ON join_t1.t1_id < join_t2.t2_id
-ORDER BY 1, 2
+ORDER BY 1, 2
----
33 44
33 55
@@ -3954,7 +3954,7 @@ query TT
explain select t1_id, t1_name, i from join_t1 t1 cross join lateral (select * from unnest(generate_series(1, t1_int))) as series(i);
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--SubqueryAlias: t1
03)----TableScan: join_t1 projection=[t1_id, t1_name]
04)--SubqueryAlias: series
@@ -4367,10 +4367,9 @@ logical_plan
physical_plan
01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)]
02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))]
-03)----ProjectionExec: expr=[]
-04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(binary_col@0, binary_col@0)]
-05)--------DataSourceExec: partitions=1, partition_sizes=[1]
-06)--------DataSourceExec: partitions=1, partition_sizes=[1]
+03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(binary_col@0, binary_col@0)], projection=[]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------DataSourceExec: partitions=1, partition_sizes=[1]
# Test hash join sort push down
# Issue: https://github.com/apache/datafusion/issues/13559
@@ -4532,7 +4531,7 @@ query TT
explain SELECT * FROM person a NATURAL JOIN lineitem b;
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--SubqueryAlias: a
03)----TableScan: person projection=[id, age, state]
04)--SubqueryAlias: b
@@ -4578,7 +4577,7 @@ query TT
explain SELECT j1_string, j2_string FROM j1, LATERAL (SELECT * FROM j2 WHERE j1_id < j2_id) AS j2;
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--TableScan: j1 projection=[j1_string]
03)--SubqueryAlias: j2
04)----Projection: j2.j2_string
@@ -4591,7 +4590,7 @@ query TT
explain SELECT * FROM j1 JOIN (j2 JOIN j3 ON(j2_id = j3_id - 2)) ON(j1_id = j2_id), LATERAL (SELECT * FROM j3 WHERE j3_string = j2_string) as j4
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--Inner Join: CAST(j2.j2_id AS Int64) = CAST(j3.j3_id AS Int64) - Int64(2)
03)----Inner Join: j1.j1_id = j2.j2_id
04)------TableScan: j1 projection=[j1_string, j1_id]
@@ -4607,11 +4606,11 @@ query TT
explain SELECT * FROM j1, LATERAL (SELECT * FROM j1, LATERAL (SELECT * FROM j2 WHERE j1_id = j2_id) as j2) as j2;
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--TableScan: j1 projection=[j1_string, j1_id]
03)--SubqueryAlias: j2
04)----Subquery:
-05)------Cross Join:
+05)------Cross Join:
06)--------TableScan: j1 projection=[j1_string, j1_id]
07)--------SubqueryAlias: j2
08)----------Subquery:
@@ -4623,7 +4622,7 @@ query TT
explain SELECT j1_string, j2_string FROM j1 LEFT JOIN LATERAL (SELECT * FROM j2 WHERE j1_id < j2_id) AS j2 ON(true);
----
logical_plan
-01)Left Join:
+01)Left Join:
02)--TableScan: j1 projection=[j1_string]
03)--SubqueryAlias: j2
04)----Projection: j2.j2_string
@@ -4636,9 +4635,9 @@ query TT
explain SELECT * FROM j1, (j2 LEFT JOIN LATERAL (SELECT * FROM j3 WHERE j1_id + j2_id = j3_id) AS j3 ON(true));
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--TableScan: j1 projection=[j1_string, j1_id]
-03)--Left Join:
+03)--Left Join:
04)----TableScan: j2 projection=[j2_string, j2_id]
05)----SubqueryAlias: j3
06)------Subquery:
@@ -4650,7 +4649,7 @@ query TT
explain SELECT * FROM j1, LATERAL (SELECT 1) AS j2;
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--TableScan: j1 projection=[j1_string, j1_id]
03)--SubqueryAlias: j2
04)----Projection: Int64(1)
@@ -4992,7 +4991,7 @@ FULL JOIN t2 ON k1 = k2
# LEFT MARK JOIN
query TT
-EXPLAIN
+EXPLAIN
SELECT *
FROM t2
WHERE k2 > 0
@@ -5067,14 +5066,14 @@ CREATE OR REPLACE TABLE t1(b INT, c INT, d INT);
statement ok
INSERT INTO t1 VALUES
- (10, 5, 3),
- ( 1, 7, 8),
- ( 2, 9, 7),
- ( 3, 8,10),
- ( 5, 6, 6),
- ( 0, 4, 9),
- ( 4, 8, 7),
- (100,6, 5);
+ (10, 5, 3),
+ ( 1, 7, 8),
+ ( 2, 9, 7),
+ ( 3, 8,10),
+ ( 5, 6, 6),
+ ( 0, 4, 9),
+ ( 4, 8, 7),
+ (100,6, 5);
query I rowsort
SELECT c
@@ -5268,3 +5267,30 @@ DROP TABLE issue_19067_left;
statement count 0
DROP TABLE issue_19067_right;
+
+# Test that empty projections pushed into joins produce correct row counts at runtime.
+# When count(1) is used over a RIGHT/FULL JOIN, the optimizer embeds an empty projection
+# (projection=[]) into the HashJoinExec. This validates that the runtime batch construction
+# handles zero-column output correctly, preserving the correct number of rows.
+
+statement ok
+CREATE TABLE empty_proj_left AS VALUES (1, 'a'), (2, 'b'), (3, 'c');
+
+statement ok
+CREATE TABLE empty_proj_right AS VALUES (1, 'x'), (2, 'y'), (4, 'z');
+
+query I
+SELECT count(1) FROM empty_proj_left RIGHT JOIN empty_proj_right ON empty_proj_left.column1 = empty_proj_right.column1;
+----
+3
+
+query I
+SELECT count(1) FROM empty_proj_left FULL JOIN empty_proj_right ON empty_proj_left.column1 = empty_proj_right.column1;
+----
+4
+
+statement count 0
+DROP TABLE empty_proj_left;
+
+statement count 0
+DROP TABLE empty_proj_right;
diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.slt
index 5a44112334..e18114bc51 100644
--- a/datafusion/sqllogictest/test_files/projection.slt
+++ b/datafusion/sqllogictest/test_files/projection.slt
@@ -167,12 +167,12 @@ set datafusion.explain.logical_plan_only = false
# project cast dictionary
query T
-SELECT
- CASE
+SELECT
+ CASE
WHEN cpu_load_short.host IS NULL THEN ''
ELSE cpu_load_short.host
END AS host
-FROM
+FROM
cpu_load_short;
----
host1
@@ -275,7 +275,6 @@ logical_plan
02)--Filter: t1.a > Int64(1)
03)----TableScan: t1 projection=[a], partial_filters=[t1.a > Int64(1)]
physical_plan
-01)ProjectionExec: expr=[]
-02)--FilterExec: a@0 > 1
-03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[]
+01)FilterExec: a@0 > 1, projection=[]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[]