Skip to content

Apache DataFusion SQL Query Engine

perf: Convert inner joins to semi joins when equivalent (#22652)

Neil Conwayb8998c762bb8
diff --git a/datafusion/optimizer/src/eliminate_join.rs b/datafusion/optimizer/src/eliminate_join.rs
index 885910c1e4..cce17c07b5 100644
--- a/datafusion/optimizer/src/eliminate_join.rs
+++ b/datafusion/optimizer/src/eliminate_join.rs
@@ -15,19 +15,135 @@
// specific language governing permissions and limitations
// under the License.
-//! [`EliminateJoin`] rewrites `INNER JOIN` with `true`/`null`
-use crate::optimizer::ApplyOrder;
+//! [`EliminateJoin`] rewrites inner joins to simpler forms to make them cheaper
+//! to evaluate. We implement two distinct rewrites:
+//!
+//! * An inner join can be rewritten to an empty relation if the join condition
+//! is trivially false.
+//!
+//! * An inner join `L ⋈ R` can be rewritten to a left semi join `L ⋉ R`
+//! (`LeftSemi`), which keeps the rows of L that have a match in R and outputs
+//! only L's columns. The rewrite to `L ⋉ R` is valid when both of the
+//! following are true:
+//!
+//! 1. None of R's columns are referenced above the join.
+//! 2. R does not observably multiply L's rows. This holds when either the
+//! join's ancestors are duplicate-insensitive (e.g., DISTINCT) or we can use
+//! functional dependencies to prove that each L row matches at most one R
+//! row (R is provably unique on the join keys).
+//!
+//! # Overview
+//!
+//! `rewrite_subtree` walks the plan top-down, threading two pieces of context
+//! down to each join:
+//!
+//! * `live` — which of the join's output columns are referenced above it. It is
+//! propagated top-down: each node asks its children only for the columns it
+//! needs from them, so a projection or aggregate asks for just the columns its
+//! expressions reference, dropping the rest (the narrowing); a join splits the
+//! set across its two inputs.
+//! * `duplicate_insensitive` — whether emitting each row once instead of many
+//! times will not change the output. A duplicate-collapsing node (e.g.,
+//! DISTINCT, GROUP BY with no aggregate functions, or the existence side of a
+//! semi/anti/mark join) sets it `true` for its subtree, and it propagates
+//! downward until a node that makes the row count observable again (a `LIMIT`,
+//! a top-N sort, ...) clears it. It is therefore fixed by the nearest such
+//! node, not by the whole ancestor chain: a collapsing node shields its subtree,
+//! so a duplicate-sensitive node further above does not matter.
+//!
+//! At each join, `rewritten_join_type` combines this context with the side's
+//! functional dependencies to choose `Inner`, `LeftSemi`, or `RightSemi`. Most
+//! node types just forward the context to their single child via
+//! `rewrite_single_input`; nodes that alter column requirements or
+//! duplicate-sensitivity (projection, aggregate, sort, ...) adjust it first.
+use crate::utils::for_each_referenced_index;
use crate::{OptimizerConfig, OptimizerRule};
-use datafusion_common::tree_node::Transformed;
-use datafusion_common::{Result, ScalarValue};
-use datafusion_expr::JoinType::Inner;
+use datafusion_common::tree_node::{Transformed, TreeNode};
+use datafusion_common::{
+ DFSchema, Dependency, HashSet, NullEquality, Result, ScalarValue,
+};
use datafusion_expr::{
- Expr,
- logical_plan::{EmptyRelation, LogicalPlan},
+ Expr, JoinType,
+ logical_plan::{
+ Aggregate, Distinct, DistinctOn, EmptyRelation, Filter, Join, Limit, LogicalPlan,
+ Partitioning, Projection, Repartition, Sort, SubqueryAlias,
+ },
};
+use std::sync::Arc;
+
+/// The columns that are "live" at a plan node, i.e., which of its output
+/// columns are referenced by an ancestor node. Represented as a set of column
+/// indices, relative to the node's schema.
+///
+/// See the module-level docs for how this set is threaded down the plan and
+/// narrowed or split at each node.
+#[derive(Debug, Default, Clone)]
+struct LiveColumns(HashSet<usize>);
+
+impl LiveColumns {
+ fn new() -> Self {
+ Self(HashSet::new())
+ }
+
+ /// Every column of `schema` is live.
+ fn all(schema: &DFSchema) -> Self {
+ Self((0..schema.fields().len()).collect())
+ }
+
+ /// The columns of `schema` referenced by any of `exprs`.
+ fn try_new<'a>(
+ exprs: impl IntoIterator<Item = &'a Expr>,
+ schema: &DFSchema,
+ ) -> Result<Self> {
+ let mut live = Self::new();
+ live.extend_from(exprs, schema)?;
+ Ok(live)
+ }
+
+ /// Inserts the index, within `schema`, of every column referenced by any of
+ /// `exprs`, including columns reached through correlated subquery outer
+ /// references.
+ fn extend_from<'a>(
+ &mut self,
+ exprs: impl IntoIterator<Item = &'a Expr>,
+ schema: &DFSchema,
+ ) -> Result<()> {
+ for expr in exprs {
+ for_each_referenced_index(expr, schema, |idx| {
+ self.0.insert(idx);
+ })?;
+ }
+ Ok(())
+ }
+
+ fn insert(&mut self, idx: usize) {
+ self.0.insert(idx);
+ }
+
+ fn is_empty(&self) -> bool {
+ self.0.is_empty()
+ }
-/// Eliminates joins when join condition is false.
-/// Replaces joins when inner join condition is true with a cross join.
+ /// Splits live columns spanning a join's combined output (the left input's
+ /// columns first, then the right input's) into the per-side sets, rebasing
+ /// the right side's indices to start at zero. `left_len` is the number of
+ /// columns contributed by the left input.
+ fn split_at(&self, left_len: usize) -> (Self, Self) {
+ let mut left = Self::new();
+ let mut right = Self::new();
+ for &idx in &self.0 {
+ if idx < left_len {
+ left.insert(idx);
+ } else {
+ right.insert(idx - left_len);
+ }
+ }
+ (left, right)
+ }
+}
+
+/// Rewrites an inner join to a semi join when one input only filters the other,
+/// and replaces an always-false inner join with an empty relation.
#[derive(Default, Debug)]
pub struct EliminateJoin;
@@ -42,44 +158,438 @@ impl OptimizerRule for EliminateJoin {
"eliminate_join"
}
- fn apply_order(&self) -> Option<ApplyOrder> {
- Some(ApplyOrder::TopDown)
- }
-
fn rewrite(
&self,
plan: LogicalPlan,
_config: &dyn OptimizerConfig,
) -> Result<Transformed<LogicalPlan>> {
- match plan {
- LogicalPlan::Join(join) if join.join_type == Inner && join.on.is_empty() => {
- match join.filter {
- Some(Expr::Literal(ScalarValue::Boolean(Some(false)), _)) => Ok(
- Transformed::yes(LogicalPlan::EmptyRelation(EmptyRelation {
- produce_one_row: false,
- schema: join.schema,
- })),
- ),
- _ => Ok(Transformed::no(LogicalPlan::Join(join))),
+ let live = LiveColumns::all(plan.schema());
+ rewrite_subtree(plan, live, false)
+ }
+}
+
+/// Rewrites `plan` and everything below it, including joins nested inside
+/// subquery expressions.
+///
+/// [`rewrite_node`] handles the node itself and recurses into its plan
+/// children; this wrapper additionally descends into the node's own subquery
+/// expressions. Each subquery is seeded as a fresh root, since its columns are
+/// independent of the enclosing plan's `live` set.
+fn rewrite_subtree(
+ plan: LogicalPlan,
+ live: LiveColumns,
+ duplicate_insensitive: bool,
+) -> Result<Transformed<LogicalPlan>> {
+ rewrite_node(plan, live, duplicate_insensitive)?.transform_data(|plan| {
+ plan.map_subqueries(|subquery| {
+ let live = LiveColumns::all(subquery.schema());
+ rewrite_subtree(subquery, live, false)
+ })
+ })
+}
+
+fn rewrite_node(
+ plan: LogicalPlan,
+ live: LiveColumns,
+ duplicate_insensitive: bool,
+) -> Result<Transformed<LogicalPlan>> {
+ match plan {
+ // The only arm that rewrites a join; the rest just thread context down to one.
+ LogicalPlan::Join(join) => rewrite_join(join, &live, duplicate_insensitive),
+ LogicalPlan::Projection(Projection {
+ expr,
+ input,
+ schema,
+ ..
+ }) => {
+ // Narrows `live` to the columns the projection's expressions reference.
+ let child_live = LiveColumns::try_new(&expr, input.schema())?;
+ rewrite_single_input(input, child_live, duplicate_insensitive, |input| {
+ Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
+ expr, input, schema,
+ )?))
+ })
+ }
+ LogicalPlan::Filter(Filter {
+ predicate, input, ..
+ }) => {
+ // Adds the predicate's columns to `live` (a side used only by the filter stays live).
+ let mut child_live = live;
+ child_live.extend_from([&predicate], input.schema())?;
+ rewrite_single_input(input, child_live, duplicate_insensitive, |input| {
+ Ok(LogicalPlan::Filter(Filter::new(predicate, input)))
+ })
+ }
+ LogicalPlan::Aggregate(Aggregate {
+ input,
+ group_expr,
+ aggr_expr,
+ schema,
+ ..
+ }) => {
+ // Narrows `live` to the grouping and aggregate expressions' columns.
+ let child_live = LiveColumns::try_new(
+ group_expr.iter().chain(&aggr_expr),
+ input.schema(),
+ )?;
+
+ // A grouping aggregate with no aggregate functions (`GROUP BY` with
+ // an empty `aggr_expr`) only observes which group-key values exist,
+ // not how many rows produced them, so its input is duplicate-
+ // insensitive.
+ let child_duplicate_insensitive =
+ !group_expr.is_empty() && aggr_expr.is_empty();
+
+ rewrite_single_input(
+ input,
+ child_live,
+ child_duplicate_insensitive,
+ |input| {
+ Ok(LogicalPlan::Aggregate(Aggregate::try_new_with_schema(
+ input, group_expr, aggr_expr, schema,
+ )?))
+ },
+ )
+ }
+ LogicalPlan::Distinct(Distinct::All(input)) => {
+ // `SELECT DISTINCT *` is equivalent to a no-aggregate `GROUP BY`
+ // over every input column, so the input is duplicate-insensitive,
+ // but every column is part of the dedup key.
+ let child_live = LiveColumns::all(input.schema());
+ rewrite_single_input(input, child_live, true, |input| {
+ Ok(LogicalPlan::Distinct(Distinct::All(input)))
+ })
+ }
+ LogicalPlan::Distinct(Distinct::On(DistinctOn {
+ on_expr,
+ select_expr,
+ sort_expr,
+ input,
+ schema,
+ })) => {
+ // `DISTINCT ON (on) select [ORDER BY sort]` is a no-aggregate
+ // `GROUP BY` on the columns it reads, so its input is duplicate-
+ // insensitive; the live columns are exactly those of the
+ // ON/SELECT/ORDER BY expressions.
+ let mut child_live =
+ LiveColumns::try_new(on_expr.iter().chain(&select_expr), input.schema())?;
+ if let Some(sort_expr) = &sort_expr {
+ child_live
+ .extend_from(sort_expr.iter().map(|s| &s.expr), input.schema())?;
+ }
+
+ rewrite_single_input(input, child_live, true, |input| {
+ Ok(LogicalPlan::Distinct(Distinct::On(DistinctOn {
+ on_expr,
+ select_expr,
+ sort_expr,
+ input,
+ schema,
+ })))
+ })
+ }
+ LogicalPlan::Sort(Sort { expr, input, fetch }) => {
+ // Adds the sort-key columns to `live`.
+ let mut child_live = live;
+ child_live.extend_from(expr.iter().map(|s| &s.expr), input.schema())?;
+
+ // A `fetch` (top-N) makes the row count observable, so duplicate-
+ // insensitivity does not survive past it.
+ let child_duplicate_insensitive = duplicate_insensitive && fetch.is_none();
+ rewrite_single_input(
+ input,
+ child_live,
+ child_duplicate_insensitive,
+ |input| Ok(LogicalPlan::Sort(Sort { expr, input, fetch })),
+ )
+ }
+ LogicalPlan::Limit(Limit { skip, fetch, input }) => {
+ // LIMIT makes the row count observable, so it clears duplicate-insensitivity.
+ rewrite_single_input(input, live, false, |input| {
+ Ok(LogicalPlan::Limit(Limit { skip, fetch, input }))
+ })
+ }
+ LogicalPlan::SubqueryAlias(SubqueryAlias { input, alias, .. }) => {
+ // Re-aliases columns 1:1, so `live` and duplicate-sensitivity pass through unchanged.
+ rewrite_single_input(input, live, duplicate_insensitive, |input| {
+ Ok(LogicalPlan::SubqueryAlias(SubqueryAlias::try_new(
+ input, alias,
+ )?))
+ })
+ }
+ LogicalPlan::Repartition(Repartition {
+ input,
+ partitioning_scheme,
+ }) => {
+ // Adds any partitioning-key columns to `live`; duplicate-sensitivity is unchanged.
+ let mut child_live = live;
+ match &partitioning_scheme {
+ Partitioning::Hash(exprs, _) | Partitioning::DistributeBy(exprs) => {
+ child_live.extend_from(exprs, input.schema())?;
}
+ Partitioning::Range(range) => {
+ child_live.extend_from(
+ range.ordering().iter().map(|sort_expr| &sort_expr.expr),
+ input.schema(),
+ )?;
+ }
+ Partitioning::RoundRobinBatch(_) => {}
}
- _ => Ok(Transformed::no(plan)),
+ rewrite_single_input(input, child_live, duplicate_insensitive, |input| {
+ Ok(LogicalPlan::Repartition(Repartition {
+ input,
+ partitioning_scheme,
+ }))
+ })
}
+ // Conservatively treat any other plan node as a fresh root, since we are
+ // not sure of its semantics with respect to duplicates or live columns.
+ _ => plan.map_children(|child| {
+ let live = LiveColumns::all(child.schema());
+ rewrite_subtree(child, live, false)
+ }),
}
+}
- fn supports_rewrite(&self) -> bool {
- true
+/// Recurses into a single-input node's child, threading `child_live` and
+/// `duplicate_insensitive` down, then rebuilds the node from the (possibly
+/// rewritten) child via `rebuild`. The child's `Transformed` flag is preserved,
+/// so the node is reported as changed exactly when its child changed.
+fn rewrite_single_input<F>(
+ input: Arc<LogicalPlan>,
+ child_live: LiveColumns,
+ duplicate_insensitive: bool,
+ rebuild: F,
+) -> Result<Transformed<LogicalPlan>>
+where
+ F: FnOnce(Arc<LogicalPlan>) -> Result<LogicalPlan>,
+{
+ rewrite_subtree(
+ Arc::unwrap_or_clone(input),
+ child_live,
+ duplicate_insensitive,
+ )?
+ .map_data(|input| rebuild(Arc::new(input)))
+}
+
+fn rewrite_join(
+ join: Join,
+ live: &LiveColumns,
+ duplicate_insensitive: bool,
+) -> Result<Transformed<LogicalPlan>> {
+ if join.join_type == JoinType::Inner
+ && join.on.is_empty()
+ && matches!(
+ join.filter.as_ref(),
+ Some(Expr::Literal(ScalarValue::Boolean(Some(false)), _))
+ )
+ {
+ return Ok(Transformed::yes(LogicalPlan::EmptyRelation(
+ EmptyRelation {
+ produce_one_row: false,
+ schema: join.schema,
+ },
+ )));
+ }
+
+ let (visible_left, visible_right) = split_join_output_columns(&join, live);
+
+ let rewritten_join_type =
+ rewritten_join_type(&join, &visible_left, &visible_right, duplicate_insensitive);
+
+ let (mut left_live, mut right_live) = match rewritten_join_type {
+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {
+ (visible_left, LiveColumns::new())
+ }
+ JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {
+ (LiveColumns::new(), visible_right)
+ }
+ _ => (visible_left, visible_right),
+ };
+
+ add_join_condition_columns(&join, &mut left_live, &mut right_live)?;
+
+ let (left_dup_insensitive, right_dup_insensitive) =
+ child_duplicate_insensitivity(rewritten_join_type, duplicate_insensitive);
+
+ let left = rewrite_subtree(
+ Arc::unwrap_or_clone(join.left),
+ left_live,
+ left_dup_insensitive,
+ )?;
+ let right = rewrite_subtree(
+ Arc::unwrap_or_clone(join.right),
+ right_live,
+ right_dup_insensitive,
+ )?;
+
+ let changed =
+ left.transformed || right.transformed || rewritten_join_type != join.join_type;
+ let left = Arc::new(left.data);
+ let right = Arc::new(right.data);
+
+ if changed {
+ // The join type or an input changed, so the output schema may have
+ // narrowed; recompute it via `try_new`.
+ Ok(Transformed::yes(LogicalPlan::Join(Join::try_new(
+ left,
+ right,
+ join.on,
+ join.filter,
+ rewritten_join_type,
+ join.join_constraint,
+ join.null_equality,
+ join.null_aware,
+ )?)))
+ } else {
+ // Nothing changed; reassemble the join reusing its existing schema rather
+ // than recomputing it.
+ Ok(Transformed::no(LogicalPlan::Join(Join {
+ left,
+ right,
+ on: join.on,
+ filter: join.filter,
+ join_type: join.join_type,
+ join_constraint: join.join_constraint,
+ schema: join.schema,
+ null_equality: join.null_equality,
+ null_aware: join.null_aware,
+ })))
}
}
+/// Returns which join inputs can safely ignore duplicate rows from their own
+/// descendants. For semi/anti/mark joins, duplicates from the existence side do
+/// not change the result even when the parent itself is duplicate-sensitive.
+fn child_duplicate_insensitivity(
+ join_type: JoinType,
+ duplicate_insensitive: bool,
+) -> (bool, bool) {
+ match join_type {
+ JoinType::Inner => (duplicate_insensitive, duplicate_insensitive),
+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {
+ (duplicate_insensitive, true)
+ }
+ JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {
+ (true, duplicate_insensitive)
+ }
+ JoinType::Left | JoinType::Right | JoinType::Full => (false, false),
+ }
+}
+
+/// Rewrites an inner join to a semi join when the removed side has no
+/// parent-visible columns and either the parent ignores duplicate output rows or
+/// the removed side is unique on the join keys.
+fn rewritten_join_type(
+ join: &Join,
+ visible_left: &LiveColumns,
+ visible_right: &LiveColumns,
+ duplicate_insensitive: bool,
+) -> JoinType {
+ if join.join_type != JoinType::Inner || join.on.is_empty() {
+ return join.join_type;
+ }
+
+ let can_remove_right = duplicate_insensitive
+ || side_unique_on_join(
+ join.right.schema(),
+ join.on.iter().map(|(_, right)| right),
+ join.null_equality,
+ );
+ if visible_right.is_empty() && can_remove_right {
+ return JoinType::LeftSemi;
+ }
+
+ let can_remove_left = duplicate_insensitive
+ || side_unique_on_join(
+ join.left.schema(),
+ join.on.iter().map(|(left, _)| left),
+ join.null_equality,
+ );
+ if visible_left.is_empty() && can_remove_left {
+ return JoinType::RightSemi;
+ }
+
+ JoinType::Inner
+}
+
+fn add_join_condition_columns(
+ join: &Join,
+ left_live: &mut LiveColumns,
+ right_live: &mut LiveColumns,
+) -> Result<()> {
+ left_live.extend_from(join.on.iter().map(|(l, _)| l), join.left.schema())?;
+ right_live.extend_from(join.on.iter().map(|(_, r)| r), join.right.schema())?;
+
+ if let Some(filter) = &join.filter {
+ left_live.extend_from([filter], join.left.schema())?;
+ right_live.extend_from([filter], join.right.schema())?;
+ }
+
+ Ok(())
+}
+
+fn split_join_output_columns(
+ join: &Join,
+ live: &LiveColumns,
+) -> (LiveColumns, LiveColumns) {
+ let left_len = join.left.schema().fields().len();
+ match join.join_type {
+ JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full => {
+ live.split_at(left_len)
+ }
+ // A semi/anti/mark join outputs only the surviving side's columns, with
+ // the same index space, so `live` passes straight through to that side.
+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {
+ (live.clone(), LiveColumns::new())
+ }
+ JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {
+ (LiveColumns::new(), live.clone())
+ }
+ }
+}
+
+fn side_unique_on_join<'a>(
+ schema: &DFSchema,
+ join_exprs: impl Iterator<Item = &'a Expr>,
+ null_equality: NullEquality,
+) -> bool {
+ let join_key_indices = join_exprs
+ .filter_map(|expr| match expr {
+ Expr::Alias(alias) => alias.expr.as_ref().try_as_col(),
+ _ => expr.try_as_col(),
+ })
+ .filter_map(|column| schema.maybe_index_of_column(column))
+ .collect::<Vec<usize>>();
+
+ schema.functional_dependencies().iter().any(|dependency| {
+ dependency.mode == Dependency::Single
+ && (!dependency.nullable || null_equality == NullEquality::NullEqualsNothing)
+ && dependency
+ .source_indices
+ .iter()
+ .all(|idx| join_key_indices.contains(idx))
+ })
+}
+
#[cfg(test)]
mod tests {
use crate::OptimizerContext;
use crate::assert_optimized_plan_eq_snapshot;
use crate::eliminate_join::EliminateJoin;
- use datafusion_common::Result;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::{
+ Constraint, Constraints, NullEquality, Result, ScalarValue, SplitPoint,
+ };
use datafusion_expr::JoinType::Inner;
- use datafusion_expr::{lit, logical_plan::builder::LogicalPlanBuilder};
+ use datafusion_expr::{
+ Expr, JoinType, Partitioning, RangePartitioning, col, exists, lit,
+ logical_plan::builder::{
+ LogicalPlanBuilder, table_scan, table_source_with_constraints,
+ },
+ out_ref_col,
+ };
+ use datafusion_functions_aggregate::expr_fn::count;
use std::sync::Arc;
macro_rules! assert_optimized_plan_equal {
@@ -110,4 +620,591 @@ mod tests {
assert_optimized_plan_equal!(plan, @"EmptyRelation: rows=0")
}
+
+ #[test]
+ fn inner_to_left_semi_when_removed_side_is_unique() -> Result<()> {
+ let plan = left_join_right_with_constraints(primary_key_on_id())?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.x
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn inner_to_left_semi_when_removed_side_is_unique_with_join_filter() -> Result<()> {
+ let right = scan("r", &test_schema(), primary_key_on_id())?;
+ let plan =
+ LogicalPlanBuilder::from(scan("l", &test_schema(), Constraints::default())?)
+ .join(
+ right,
+ Inner,
+ (vec!["l.id"], vec!["r.id"]),
+ Some(col("r.y").gt(col("l.x"))),
+ )?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.x
+ LeftSemi Join: l.id = r.id Filter: r.y > l.x
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn inner_to_right_semi_when_removed_side_is_unique() -> Result<()> {
+ let plan = left_with_constraints_join_right(primary_key_on_id())?
+ .project(vec![col("r.y")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: r.y
+ RightSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn inner_to_left_semi_for_duplicate_insensitive_parent() -> Result<()> {
+ let plan = left_join_right()?
+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[]]
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn aggregate_with_aggregates_is_not_duplicate_insensitive() -> Result<()> {
+ // A `GROUP BY` *with* aggregate functions observes how many rows fall in
+ // each group, so its input is not duplicate-insensitive. With a non-unique
+ // right side the join must stay an inner join: collapsing it to a semi
+ // join would drop matching duplicates and undercount `count(l.id)`.
+ let plan = left_join_right()?
+ .aggregate(vec![col("l.x")], vec![count(col("l.id"))])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[count(l.id)]]
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn duplicate_insensitive_context_propagates_through_join_tree() -> Result<()> {
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let middle = scan("m", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), Constraints::default())?;
+
+ let left_join_middle = LogicalPlanBuilder::from(left)
+ .join(middle, Inner, (vec!["l.id"], vec!["m.id"]), None)?
+ .build()?;
+
+ let plan = LogicalPlanBuilder::from(left_join_middle)
+ .join(right, Inner, (vec!["l.id"], vec!["r.id"]), None)?
+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[]]
+ LeftSemi Join: l.id = r.id
+ LeftSemi Join: l.id = m.id
+ TableScan: l
+ TableScan: m
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn projection_does_not_rewrite_without_uniqueness() -> Result<()> {
+ let plan = left_join_right()?.project(vec![col("l.x")])?.build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.x
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn required_filter_column_prevents_duplicate_insensitive_rewrite() -> Result<()> {
+ let plan = left_join_right()?
+ .filter(col("r.y").gt(lit(10_i32)))?
+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[]]
+ Filter: r.y > Int32(10)
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn distinct_star_keeps_unreferenced_side() -> Result<()> {
+ // `SELECT DISTINCT *` deduplicates on every join-output column, including
+ // the right side's. With a non-unique right side the inner join can
+ // multiply left rows into distinct `(l, r)` combinations, so the join
+ // must not be rewritten to a semi join (which would drop the right
+ // columns from the DISTINCT key and undercount the result). This holds
+ // even when the right side is unique on the join keys: its columns are
+ // part of the DISTINCT key regardless.
+ let plan = left_join_right()?
+ .distinct()?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.x
+ Distinct:
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn distinct_drops_unreferenced_side_when_projected() -> Result<()> {
+ // `SELECT DISTINCT l.x` projects the right side away below the DISTINCT,
+ // leaving it outside the dedup key. Like a no-aggregate `GROUP BY l.x`,
+ // the DISTINCT makes the input duplicate-insensitive, so the inner join
+ // collapses to a semi join even though the right side is not unique.
+ let plan = left_join_right()?
+ .project(vec![col("l.x")])?
+ .distinct()?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Distinct:
+ Projection: l.x
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn correlated_subquery_outer_ref_prevents_rewrite() -> Result<()> {
+ // The aggregate makes the parent duplicate-insensitive, so absent any
+ // other use of the right side the join would collapse to a semi join.
+ // But the `EXISTS` subquery correlates on `r.y`, so the right side is
+ // still needed and the join must stay an inner join. Otherwise the
+ // semi join would drop `r`, orphaning the correlated `r.y` reference.
+ let subquery =
+ LogicalPlanBuilder::from(scan("s", &test_schema(), Constraints::default())?)
+ .filter(col("s.id").eq(out_ref_col(DataType::Int32, "r.y")))?
+ .project(vec![lit(1)])?
+ .build()?;
+
+ let plan = left_join_right()?
+ .filter(exists(Arc::new(subquery)))?
+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[]]
+ Filter: EXISTS (<subquery>)
+ Subquery:
+ Projection: Int32(1)
+ Filter: s.id = outer_ref(r.y)
+ TableScan: s
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn inner_to_semi_inside_uncorrelated_subquery() -> Result<()> {
+ // A join nested inside a (not-yet-decorrelated) subquery is still
+ // rewritten, because `rewrite_subtree` descends into subquery plans
+ // itself via `map_subqueries`. Here the subquery's projection keeps
+ // only `l.x` and the removed side `r` is unique (PK), so the inner join
+ // collapses to a semi join.
+ let subquery = left_join_right_with_constraints(primary_key_on_id())?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ let plan = LogicalPlanBuilder::from(scan(
+ "outer",
+ &test_schema(),
+ Constraints::default(),
+ )?)
+ .filter(exists(Arc::new(subquery)))?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Filter: EXISTS (<subquery>)
+ Subquery:
+ Projection: l.x
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ TableScan: outer
+ ")
+ }
+
+ #[test]
+ fn inner_to_semi_inside_correlated_subquery() -> Result<()> {
+ // `map_subqueries` descends into correlated subqueries too, not just
+ // uncorrelated ones, so a join inside one is still rewritten. The
+ // subquery correlates on `outer.id` (via the filter), but that reference
+ // and the projection touch only `l`; `r` is unique (PK) and unreferenced,
+ // so the inner join inside the subquery collapses to a semi join.
+ let subquery = left_join_right_with_constraints(primary_key_on_id())?
+ .filter(col("l.x").eq(out_ref_col(DataType::Int32, "outer.id")))?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ let plan = LogicalPlanBuilder::from(scan(
+ "outer",
+ &test_schema(),
+ Constraints::default(),
+ )?)
+ .filter(exists(Arc::new(subquery)))?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Filter: EXISTS (<subquery>)
+ Subquery:
+ Projection: l.x
+ Filter: l.x = outer_ref(outer.id)
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ TableScan: outer
+ ")
+ }
+
+ #[test]
+ fn nullable_unique_rewrites_under_null_equals_nothing() -> Result<()> {
+ // A `UNIQUE` (rather than `PRIMARY KEY`) constraint marks the key as
+ // nullable. Under the default `NullEqualsNothing` join semantics a null
+ // key matches nothing, so a unique side still yields at most one match
+ // per left row and the inner join can become a semi join.
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), unique_on_x())?;
+ let plan = LogicalPlanBuilder::from(left)
+ .join(right, Inner, (vec!["l.x"], vec!["r.x"]), None)?
+ .project(vec![col("l.id")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.id
+ LeftSemi Join: l.x = r.x
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn nullable_unique_does_not_rewrite_under_null_equals_null() -> Result<()> {
+ // With `NullEqualsNull` semantics two null keys compare equal, so a
+ // nullable `UNIQUE` key no longer guarantees at most one match per left
+ // row: several null-keyed right rows could match a null-keyed left row.
+ // Uniqueness on the join keys is therefore not established and the inner
+ // join must be preserved.
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), unique_on_x())?;
+ let plan = LogicalPlanBuilder::from(left)
+ .join_detailed(
+ right,
+ Inner,
+ (vec!["l.x"], vec!["r.x"]),
+ None,
+ NullEquality::NullEqualsNull,
+ )?
+ .project(vec![col("l.id")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.id
+ Inner Join: l.x = r.x
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn composite_unique_rewrites_when_join_covers_all_key_columns() -> Result<()> {
+ // The removed side is unique on the composite key `(id, x)`. The join
+ // equates both key columns, so each left row matches at most one right
+ // row and the inner join can become a semi join.
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), composite_primary_key_on_id_x())?;
+ let plan = LogicalPlanBuilder::from(left)
+ .join(
+ right,
+ Inner,
+ (vec!["l.id", "l.x"], vec!["r.id", "r.x"]),
+ None,
+ )?
+ .project(vec![col("l.y")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.y
+ LeftSemi Join: l.id = r.id, l.x = r.x
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn composite_unique_does_not_rewrite_when_join_misses_a_key_column() -> Result<()> {
+ // The removed side is unique only on the *composite* key `(id, x)`. The
+ // join equates `id` but not `x`, so a left row may match many right rows
+ // (those sharing its `id` but differing in `x`). Uniqueness on the join
+ // keys is not established, so the inner join must be preserved. This
+ // guards the requirement that the join cover *every* column of the
+ // unique key, not just some.
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), composite_primary_key_on_id_x())?;
+ let plan = LogicalPlanBuilder::from(left)
+ .join(right, Inner, (vec!["l.id"], vec!["r.id"]), None)?
+ .project(vec![col("l.y")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.y
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn top_n_sort_blocks_duplicate_insensitive_rewrite() -> Result<()> {
+ // A top-N `Sort` (one with a `fetch`) makes the row count observable, so
+ // the duplicate-insensitivity established by the `GROUP BY` does not survive
+ // past it. With a non-unique right side the join must stay an inner join: a
+ // semi join could drop matching duplicates and change which rows fall within
+ // the top N.
+ let plan = left_join_right()?
+ .sort_with_limit(vec![col("l.x").sort(true, false)], Some(5))?
+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[]]
+ Sort: l.x ASC NULLS LAST, fetch=5
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn sort_without_fetch_preserves_duplicate_insensitive_rewrite() -> Result<()> {
+ // A `Sort` without a `fetch` does not make the row count observable, so it
+ // forwards the parent's duplicate-insensitivity to the join unchanged
+ // (sorting before or after duplicate removal is equivalent). The non-unique
+ // right side is unreferenced, so the inner join collapses to a semi join.
+ let plan = left_join_right()?
+ .sort(vec![col("l.x").sort(true, false)])?
+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[]]
+ Sort: l.x ASC NULLS LAST
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn limit_blocks_duplicate_insensitive_rewrite() -> Result<()> {
+ // `LIMIT` makes the row count observable, clearing the duplicate-
+ // insensitivity established by the `GROUP BY`. With a non-unique right side
+ // the join must stay an inner join, since a semi join could drop matching
+ // duplicates and change which rows the limit returns.
+ let plan = left_join_right()?
+ .limit(0, Some(5))?
+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[l.x]], aggr=[[]]
+ Limit: skip=0, fetch=5
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn repartition_hash_key_keeps_removed_side_live() -> Result<()> {
+ // The projection keeps only `l.x`, and the right side is unique (PK), so
+ // absent any other use of `r` the inner join would collapse to a semi join.
+ // But the `Repartition` hashes on `r.y`, which keeps the right side live, so
+ // the join must stay an inner join to preserve `r.y` for the partitioning.
+ let plan = left_join_right_with_constraints(primary_key_on_id())?
+ .repartition(Partitioning::Hash(vec![col("r.y")], 4))?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.x
+ Repartition: Hash(r.y) partition_count=4
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn repartition_range_key_keeps_removed_side_live() -> Result<()> {
+ // The projection keeps only `l.x`, and the right side is unique (PK), so
+ // absent any other use of `r` the inner join would collapse to a semi join.
+ // But the `Repartition` ranges on `r.y`, which keeps the right side live, so
+ // the join must stay an inner join to preserve `r.y` for the partitioning.
+ let plan = left_join_right_with_constraints(primary_key_on_id())?
+ .repartition(Partitioning::Range(RangePartitioning::try_new(
+ vec![col("r.y").sort(true, true)],
+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],
+ )?))?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.x
+ Repartition: Range([r.y ASC NULLS FIRST], [(10)], 2)
+ Inner Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn distinct_on_enables_semi_join_rewrite() -> Result<()> {
+ // `DISTINCT ON (l.x)` is a no-aggregate `GROUP BY` on the columns it reads,
+ // so it makes its input duplicate-insensitive. The non-unique right side is
+ // unreferenced, so the inner join collapses to a semi join.
+ let plan = left_join_right()?
+ .distinct_on(vec![col("l.x")], vec![col("l.x")], None)?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ DistinctOn: on_expr=[[l.x]], select_expr=[[l.x]], sort_expr=[[]]
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn existing_semi_join_passes_through_unchanged() -> Result<()> {
+ // A join that is already a semi join is threaded through unchanged: the rule
+ // only rewrites inner joins. This exercises the context-propagation paths for
+ // a non-inner join type, whose existence side contributes no live columns.
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), Constraints::default())?;
+ let plan = LogicalPlanBuilder::from(left)
+ .join(
+ right,
+ JoinType::LeftSemi,
+ (vec!["l.id"], vec!["r.id"]),
+ None,
+ )?
+ .project(vec![col("l.x")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: l.x
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ fn left_join_right() -> Result<LogicalPlanBuilder> {
+ left_join_right_with_constraints(Constraints::default())
+ }
+
+ fn left_join_right_with_constraints(
+ right_constraints: Constraints,
+ ) -> Result<LogicalPlanBuilder> {
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), right_constraints)?;
+
+ LogicalPlanBuilder::from(left).join(
+ right,
+ Inner,
+ (vec!["l.id"], vec!["r.id"]),
+ None,
+ )
+ }
+
+ fn left_with_constraints_join_right(
+ left_constraints: Constraints,
+ ) -> Result<LogicalPlanBuilder> {
+ let left = scan("l", &test_schema(), left_constraints)?;
+ let right = scan("r", &test_schema(), Constraints::default())?;
+
+ LogicalPlanBuilder::from(left).join(
+ right,
+ Inner,
+ (vec!["l.id"], vec!["r.id"]),
+ None,
+ )
+ }
+
+ fn scan(
+ name: &str,
+ schema: &Schema,
+ constraints: Constraints,
+ ) -> Result<datafusion_expr::logical_plan::LogicalPlan> {
+ if constraints.is_empty() {
+ table_scan(Some(name), schema, None)?.build()
+ } else {
+ LogicalPlanBuilder::scan(
+ name,
+ table_source_with_constraints(schema, constraints),
+ None,
+ )?
+ .build()
+ }
+ }
+
+ fn test_schema() -> Schema {
+ Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("x", DataType::Int32, true),
+ Field::new("y", DataType::Int32, true),
+ ])
+ }
+
+ fn primary_key_on_id() -> Constraints {
+ Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])])
+ }
+
+ /// A nullable unique key on column `x` (index 1). `Unique` (unlike
+ /// `PrimaryKey`) marks the dependency as nullable, which is what gates the
+ /// rewrite on the join's `null_equality`.
+ fn unique_on_x() -> Constraints {
+ Constraints::new_unverified(vec![Constraint::Unique(vec![1])])
+ }
+
+ /// A composite primary key spanning columns `id` and `x` (indices 0 and 1).
+ fn composite_primary_key_on_id_x() -> Constraints {
+ Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0, 1])])
+ }
}
diff --git a/datafusion/optimizer/src/optimize_projections/required_indices.rs b/datafusion/optimizer/src/optimize_projections/required_indices.rs
index 5e73a9fbec..33f0d48721 100644
--- a/datafusion/optimizer/src/optimize_projections/required_indices.rs
+++ b/datafusion/optimizer/src/optimize_projections/required_indices.rs
@@ -17,7 +17,8 @@
//! [`RequiredIndices`] helper for OptimizeProjection
-use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
+use crate::utils::for_each_referenced_index;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Column, DFSchemaRef, Result};
use datafusion_expr::{Expr, LogicalPlan};
@@ -112,29 +113,8 @@ impl RequiredIndices {
/// * `input_schema`: The input schema to analyze for index requirements.
/// * `expr`: An expression for which we want to find necessary field indices.
fn add_expr(&mut self, input_schema: &DFSchemaRef, expr: &Expr) {
- // `apply` does not descend into subqueries, so recurse manually to
- // handle those cases.
- expr.apply(|e| {
- match e {
- Expr::Column(c) | Expr::OuterReferenceColumn(_, c) => {
- if let Some(idx) = input_schema.maybe_index_of_column(c) {
- self.indices.push(idx);
- }
- }
- Expr::ScalarSubquery(sub) => {
- self.add_exprs(input_schema, &sub.outer_ref_columns);
- }
- Expr::Exists(ex) => {
- self.add_exprs(input_schema, &ex.subquery.outer_ref_columns);
- }
- Expr::InSubquery(isq) => {
- self.add_exprs(input_schema, &isq.subquery.outer_ref_columns);
- }
- _ => {}
- }
- Ok(TreeNodeRecursion::Continue)
- })
- .expect("traversal is infallible");
+ for_each_referenced_index(expr, input_schema, |idx| self.indices.push(idx))
+ .expect("traversal is infallible");
}
/// Like [`Self::add_expr`], but for multiple expressions.
diff --git a/datafusion/optimizer/src/utils.rs b/datafusion/optimizer/src/utils.rs
index ad151d1ddb..b29649e9ea 100644
--- a/datafusion/optimizer/src/utils.rs
+++ b/datafusion/optimizer/src/utils.rs
@@ -24,9 +24,10 @@ use arrow::array::{Array, RecordBatch, new_null_array};
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_common::TableReference;
use datafusion_common::cast::as_boolean_array;
-use datafusion_common::tree_node::{TransformedResult, TreeNode};
+use datafusion_common::tree_node::{TransformedResult, TreeNode, TreeNodeRecursion};
use datafusion_common::{Column, DFSchema, Result, ScalarValue};
use datafusion_expr::execution_props::ExecutionProps;
+use datafusion_expr::expr::{Exists, InSubquery, SetComparison};
use datafusion_expr::expr_rewriter::replace_col;
use datafusion_expr::{ColumnarValue, Expr, logical_plan::LogicalPlan};
use datafusion_physical_expr::create_physical_expr;
@@ -37,6 +38,56 @@ use std::sync::Arc;
/// as it was initially placed here and then moved elsewhere.
pub use datafusion_expr::expr_rewriter::NamePreserver;
+/// Invokes `f` with the index, within `schema`, of every column referenced by
+/// `expr` — including columns reached through a correlated subquery's outer
+/// references. Columns absent from `schema` are skipped.
+///
+/// A subquery's own plan is intentionally not traversed: its internal columns
+/// index into its own schema, not `schema`; only the outer (correlated) columns
+/// it references from `schema` are relevant. The comparison expression of an
+/// `IN`/set-comparison subquery is reached by the normal expression walk.
+///
+/// This is the shared primitive behind the top-down "which of a node's output
+/// columns does an ancestor still need" analyses, namely
+/// [`OptimizeProjections`](crate::optimize_projections::OptimizeProjections)
+/// and [`EliminateJoin`](crate::eliminate_join::EliminateJoin). The two keep
+/// their own required-index containers (an ordered set vs. a hash set), so this
+/// reports indices through a callback rather than populating a shared type.
+pub(crate) fn for_each_referenced_index(
+ expr: &Expr,
+ schema: &DFSchema,
+ mut f: impl FnMut(usize),
+) -> Result<()> {
+ visit_referenced_indices(expr, schema, &mut f)
+}
+
+fn visit_referenced_indices(
+ expr: &Expr,
+ schema: &DFSchema,
+ f: &mut dyn FnMut(usize),
+) -> Result<()> {
+ expr.apply(|expr| {
+ match expr {
+ Expr::Column(column) | Expr::OuterReferenceColumn(_, column) => {
+ if let Some(idx) = schema.maybe_index_of_column(column) {
+ f(idx);
+ }
+ }
+ Expr::Exists(Exists { subquery, .. })
+ | Expr::InSubquery(InSubquery { subquery, .. })
+ | Expr::SetComparison(SetComparison { subquery, .. })
+ | Expr::ScalarSubquery(subquery) => {
+ for outer in &subquery.outer_ref_columns {
+ visit_referenced_indices(outer, schema, f)?;
+ }
+ }
+ _ => {}
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ Ok(())
+}
+
/// Returns true if `expr` contains all columns in `schema_cols`
pub(crate) fn has_all_column_refs(
expr: &Expr,
diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt
index b037aef3c2..9be1d39d63 100644
--- a/datafusion/sqllogictest/test_files/joins.slt
+++ b/datafusion/sqllogictest/test_files/joins.slt
@@ -1333,19 +1333,57 @@ inner join join_t2 on join_t1.t1_id = join_t2.t2_id
----
logical_plan
01)Aggregate: groupBy=[[join_t1.t1_id]], aggr=[[]]
-02)--Projection: join_t1.t1_id
-03)----Inner Join: join_t1.t1_id = join_t2.t2_id
-04)------TableScan: join_t1 projection=[t1_id]
-05)------TableScan: join_t2 projection=[t2_id]
+02)--LeftSemi Join: join_t1.t1_id = join_t2.t2_id
+03)----TableScan: join_t1 projection=[t1_id]
+04)----TableScan: join_t2 projection=[t2_id]
physical_plan
01)AggregateExec: mode=FinalPartitioned, gby=[t1_id@0 as t1_id], aggr=[]
02)--RepartitionExec: partitioning=Hash([t1_id@0], 2), input_partitions=2
03)----AggregateExec: mode=Partial, gby=[t1_id@0 as t1_id], aggr=[]
-04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1_id@0, t2_id@0)], projection=[t1_id@0]
+04)------HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(t1_id@0, t2_id@0)]
05)--------DataSourceExec: partitions=1, partition_sizes=[1]
06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
07)----------DataSourceExec: partitions=1, partition_sizes=[1]
+statement ok
+set datafusion.explain.logical_plan_only = true;
+
+# A single `count(DISTINCT col)` over a join whose other side is used only as an
+# existence filter can be rewritten to a semi join.
+query TT
+EXPLAIN
+select join_t1.t1_id, count(distinct join_t1.t1_int)
+from join_t1
+inner join join_t2 on join_t1.t1_id = join_t2.t2_id
+group by join_t1.t1_id
+----
+logical_plan
+01)Projection: join_t1.t1_id, count(alias1) AS count(DISTINCT join_t1.t1_int)
+02)--Aggregate: groupBy=[[join_t1.t1_id]], aggr=[[count(alias1)]]
+03)----Aggregate: groupBy=[[join_t1.t1_id, join_t1.t1_int AS alias1]], aggr=[[]]
+04)------LeftSemi Join: join_t1.t1_id = join_t2.t2_id
+05)--------TableScan: join_t1 projection=[t1_id, t1_int]
+06)--------TableScan: join_t2 projection=[t2_id]
+
+# A similar query with two DISTINCT aggregates is currently not rewritten
+# TODO: https://github.com/apache/datafusion/issues/22644
+query TT
+EXPLAIN
+select join_t1.t1_id, count(distinct join_t1.t1_int), count(distinct join_t1.t1_name)
+from join_t1
+inner join join_t2 on join_t1.t1_id = join_t2.t2_id
+group by join_t1.t1_id
+----
+logical_plan
+01)Aggregate: groupBy=[[join_t1.t1_id]], aggr=[[count(DISTINCT join_t1.t1_int), count(DISTINCT join_t1.t1_name)]]
+02)--Projection: join_t1.t1_id, join_t1.t1_name, join_t1.t1_int
+03)----Inner Join: join_t1.t1_id = join_t2.t2_id
+04)------TableScan: join_t1 projection=[t1_id, t1_name, t1_int]
+05)------TableScan: join_t2 projection=[t2_id]
+
+statement ok
+set datafusion.explain.logical_plan_only = false;
+
# Join on struct
query TT
explain select join_t3.s3, join_t4.s4
@@ -1411,10 +1449,9 @@ logical_plan
01)Projection: count(alias1) AS count(DISTINCT join_t1.t1_id)
02)--Aggregate: groupBy=[[]], aggr=[[count(alias1)]]
03)----Aggregate: groupBy=[[join_t1.t1_id AS alias1]], aggr=[[]]
-04)------Projection: join_t1.t1_id
-05)--------Inner Join: join_t1.t1_id = join_t2.t2_id
-06)----------TableScan: join_t1 projection=[t1_id]
-07)----------TableScan: join_t2 projection=[t2_id]
+04)------LeftSemi Join: join_t1.t1_id = join_t2.t2_id
+05)--------TableScan: join_t1 projection=[t1_id]
+06)--------TableScan: join_t2 projection=[t2_id]
physical_plan
01)ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT join_t1.t1_id)]
02)--AggregateExec: mode=Final, gby=[], aggr=[count(alias1)]
@@ -1423,7 +1460,7 @@ physical_plan
05)--------AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[]
06)----------RepartitionExec: partitioning=Hash([alias1@0], 2), input_partitions=2
07)------------AggregateExec: mode=Partial, gby=[t1_id@0 as alias1], aggr=[]
-08)--------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1_id@0, t2_id@0)], projection=[t1_id@0]
+08)--------------HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(t1_id@0, t2_id@0)]
09)----------------DataSourceExec: partitions=1, partition_sizes=[1]
10)----------------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
11)------------------DataSourceExec: partitions=1, partition_sizes=[1]
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index 3d6f802745..d305109a48 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -338,13 +338,13 @@ where c_acctbal < (
logical_plan
01)Sort: customer.c_custkey ASC NULLS LAST
02)--Projection: customer.c_custkey
-03)----Inner Join: customer.c_custkey = __scalar_sq_1.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_1.sum(orders.o_totalprice)
+03)----LeftSemi Join: customer.c_custkey = __scalar_sq_1.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_1.sum(orders.o_totalprice)
04)------TableScan: customer projection=[c_custkey, c_acctbal]
05)------SubqueryAlias: __scalar_sq_1
06)--------Projection: sum(orders.o_totalprice), orders.o_custkey
07)----------Aggregate: groupBy=[[orders.o_custkey]], aggr=[[sum(orders.o_totalprice)]]
08)------------Projection: orders.o_custkey, orders.o_totalprice
-09)--------------Inner Join: orders.o_orderkey = __scalar_sq_2.l_orderkey Filter: CAST(orders.o_totalprice AS Decimal128(25, 2)) < __scalar_sq_2.price
+09)--------------LeftSemi Join: orders.o_orderkey = __scalar_sq_2.l_orderkey Filter: CAST(orders.o_totalprice AS Decimal128(25, 2)) < __scalar_sq_2.price
10)----------------TableScan: orders projection=[o_orderkey, o_custkey, o_totalprice]
11)----------------SubqueryAlias: __scalar_sq_2
12)------------------Projection: sum(lineitem.l_extendedprice) AS price, lineitem.l_orderkey
@@ -555,7 +555,7 @@ logical_plan
02)--TableScan: t0 projection=[t0_id, t0_name]
03)--SubqueryAlias: __correlated_sq_2
04)----Projection: t1.t1_name
-05)------Inner Join: t1.t1_id = t2.t2_id
+05)------LeftSemi Join: t1.t1_id = t2.t2_id
06)--------TableScan: t1 projection=[t1_id, t1_name]
07)--------TableScan: t2 projection=[t2_id]
@@ -568,7 +568,7 @@ logical_plan
02)--TableScan: t0 projection=[t0_id, t0_name]
03)--SubqueryAlias: __correlated_sq_1
04)----Projection: t2.t2_name
-05)------Inner Join: t1.t1_id = t2.t2_id
+05)------RightSemi Join: t1.t1_id = t2.t2_id
06)--------TableScan: t1 projection=[t1_id]
07)--------SubqueryAlias: t2
08)----------TableScan: t2 projection=[t2_id, t2_name]
@@ -1675,7 +1675,7 @@ where c_acctbal < (
logical_plan
01)Sort: customer.c_custkey ASC NULLS LAST
02)--Projection: customer.c_custkey
-03)----Inner Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)
+03)----LeftSemi Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)
04)------TableScan: customer projection=[c_custkey, c_acctbal]
05)------SubqueryAlias: __scalar_sq_2
06)--------Projection: sum(orders.o_totalprice), orders.o_custkey
@@ -1701,7 +1701,7 @@ where c_acctbal < (
logical_plan
01)Sort: customer.c_custkey ASC NULLS LAST
02)--Projection: customer.c_custkey
-03)----Inner Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)
+03)----LeftSemi Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)
04)------TableScan: customer projection=[c_custkey, c_acctbal]
05)------SubqueryAlias: __scalar_sq_2
06)--------Projection: sum(orders.o_totalprice), orders.o_custkey
@@ -1746,7 +1746,7 @@ WHERE e1.salary > (
----
logical_plan
01)Projection: e1.employee_name, e1.salary
-02)--Inner Join: e1.dept_id = __scalar_sq_1.dept_id Filter: CAST(e1.salary AS Decimal128(38, 14)) > __scalar_sq_1.avg(e2.salary)
+02)--LeftSemi Join: e1.dept_id = __scalar_sq_1.dept_id Filter: CAST(e1.salary AS Decimal128(38, 14)) > __scalar_sq_1.avg(e2.salary)
03)----SubqueryAlias: e1
04)------TableScan: employees projection=[employee_name, dept_id, salary]
05)----SubqueryAlias: __scalar_sq_1
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part
index 6bab765c67..cd86b618f0 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part
@@ -54,7 +54,7 @@ logical_plan
05)--------Projection: CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty) AS Float64) * Float64(0.0001) AS Decimal128(38, 15))
06)----------Aggregate: groupBy=[[]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]
07)------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost
-08)--------------Inner Join: supplier.s_nationkey = nation.n_nationkey
+08)--------------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey
09)----------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey
10)------------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey
11)--------------------TableScan: partsupp projection=[ps_suppkey, ps_availqty, ps_supplycost]
@@ -64,7 +64,7 @@ logical_plan
15)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY")]
16)------Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]
17)--------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost
-18)----------Inner Join: supplier.s_nationkey = nation.n_nationkey
+18)----------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey
19)------------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey
20)--------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey
21)----------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost]
@@ -81,7 +81,7 @@ physical_plan
06)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
07)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4
08)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
-09)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]
+09)----------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]
10)------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=4
11)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]
12)----------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4
@@ -96,7 +96,7 @@ physical_plan
21)----AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
22)------CoalescePartitionsExec
23)--------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
-24)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]
+24)----------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]
25)------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4
26)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]
27)----------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part
index 9526d85319..7ef36a72ec 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part
@@ -57,7 +57,7 @@ logical_plan
01)Projection: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS revenue
02)--Aggregate: groupBy=[[]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(1,20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]
03)----Projection: lineitem.l_extendedprice, lineitem.l_discount
-04)------Inner Join: lineitem.l_partkey = part.p_partkey Filter: part.p_brand = Utf8View("Brand#12") AND part.p_container IN ([Utf8View("SM CASE"), Utf8View("SM BOX"), Utf8View("SM PACK"), Utf8View("SM PKG")]) AND lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_container IN ([Utf8View("MED BAG"), Utf8View("MED BOX"), Utf8View("MED PKG"), Utf8View("MED PACK")]) AND lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_container IN ([Utf8View("LG CASE"), Utf8View("LG BOX"), Utf8View("LG PACK"), Utf8View("LG PKG")]) AND lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2) AND part.p_size <= Int32(15)
+04)------LeftSemi Join: lineitem.l_partkey = part.p_partkey Filter: part.p_brand = Utf8View("Brand#12") AND part.p_container IN ([Utf8View("SM CASE"), Utf8View("SM BOX"), Utf8View("SM PACK"), Utf8View("SM PKG")]) AND lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_container IN ([Utf8View("MED BAG"), Utf8View("MED BOX"), Utf8View("MED PKG"), Utf8View("MED PACK")]) AND lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_container IN ([Utf8View("LG CASE"), Utf8View("LG BOX"), Utf8View("LG PACK"), Utf8View("LG PKG")]) AND lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2) AND part.p_size <= Int32(15)
05)--------Projection: lineitem.l_partkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount
06)----------Filter: (lineitem.l_shipmode = Utf8View("AIR") OR lineitem.l_shipmode = Utf8View("AIR REG")) AND lineitem.l_shipinstruct = Utf8View("DELIVER IN PERSON") AND (lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) OR lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) OR lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2))
07)------------TableScan: lineitem projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View("AIR") OR lineitem.l_shipmode = Utf8View("AIR REG"), lineitem.l_shipinstruct = Utf8View("DELIVER IN PERSON"), lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) OR lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) OR lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2)]
@@ -68,7 +68,7 @@ physical_plan
02)--AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
03)----CoalescePartitionsExec
04)------AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= 1.00 AND l_quantity@0 <= 11.00 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= 10.00 AND l_quantity@0 <= 20.00 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= 20.00 AND l_quantity@0 <= 30.00 AND p_size@2 <= 15, projection=[l_extendedprice@2, l_discount@3]
+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= 1.00 AND l_quantity@0 <= 11.00 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= 10.00 AND l_quantity@0 <= 20.00 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= 20.00 AND l_quantity@0 <= 30.00 AND p_size@2 <= 15, projection=[l_extendedprice@2, l_discount@3]
06)----------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=4
07)------------FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= 1.00 AND l_quantity@1 <= 11.00 OR l_quantity@1 >= 10.00 AND l_quantity@1 <= 20.00 OR l_quantity@1 >= 20.00 AND l_quantity@1 <= 30.00), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3]
08)--------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], constraints=[PrimaryKey([0, 3])], file_type=csv, has_header=false
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part
index 31702ab39e..b6fa1c4806 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part
@@ -65,9 +65,9 @@ limit 10;
logical_plan
01)Sort: supplier.s_acctbal DESC NULLS FIRST, nation.n_name ASC NULLS LAST, supplier.s_name ASC NULLS LAST, part.p_partkey ASC NULLS LAST, fetch=10
02)--Projection: supplier.s_acctbal, supplier.s_name, nation.n_name, part.p_partkey, part.p_mfgr, supplier.s_address, supplier.s_phone, supplier.s_comment
-03)----Inner Join: part.p_partkey = __scalar_sq_1.ps_partkey, partsupp.ps_supplycost = __scalar_sq_1.min(partsupp.ps_supplycost)
+03)----LeftSemi Join: part.p_partkey = __scalar_sq_1.ps_partkey, partsupp.ps_supplycost = __scalar_sq_1.min(partsupp.ps_supplycost)
04)------Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_supplycost, nation.n_name
-05)--------Inner Join: nation.n_regionkey = region.r_regionkey
+05)--------LeftSemi Join: nation.n_regionkey = region.r_regionkey
06)----------Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_supplycost, nation.n_name, nation.n_regionkey
07)------------Inner Join: supplier.s_nationkey = nation.n_nationkey
08)--------------Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_nationkey, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_supplycost
@@ -87,7 +87,7 @@ logical_plan
22)--------Projection: min(partsupp.ps_supplycost), partsupp.ps_partkey
23)----------Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[min(partsupp.ps_supplycost)]]
24)------------Projection: partsupp.ps_partkey, partsupp.ps_supplycost
-25)--------------Inner Join: nation.n_regionkey = region.r_regionkey
+25)--------------LeftSemi Join: nation.n_regionkey = region.r_regionkey
26)----------------Projection: partsupp.ps_partkey, partsupp.ps_supplycost, nation.n_regionkey
27)------------------Inner Join: supplier.s_nationkey = nation.n_nationkey
28)--------------------Projection: partsupp.ps_partkey, partsupp.ps_supplycost, supplier.s_nationkey
@@ -101,9 +101,9 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], fetch=10
02)--SortExec: TopK(fetch=10), expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true]
-03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[s_acctbal@5, s_name@2, n_name@8, p_partkey@0, p_mfgr@1, s_address@3, s_phone@4, s_comment@6]
+03)----HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[s_acctbal@5, s_name@2, n_name@8, p_partkey@0, p_mfgr@1, s_address@3, s_phone@4, s_comment@6]
04)------RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 4), input_partitions=4
-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@9, r_regionkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, ps_supplycost@7, n_name@8]
+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@9, r_regionkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, ps_supplycost@7, n_name@8]
06)----------RepartitionExec: partitioning=Hash([n_regionkey@9], 4), input_partitions=4
07)------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@4, n_nationkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@10, n_regionkey@11]
08)--------------RepartitionExec: partitioning=Hash([s_nationkey@4], 4), input_partitions=4
@@ -128,7 +128,7 @@ physical_plan
27)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]
28)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4
29)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]
-30)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@2, r_regionkey@0)], projection=[ps_partkey@0, ps_supplycost@1]
+30)----------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@2, r_regionkey@0)], projection=[ps_partkey@0, ps_supplycost@1]
31)------------------RepartitionExec: partitioning=Hash([n_regionkey@2], 4), input_partitions=4
32)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_partkey@0, ps_supplycost@1, n_regionkey@4]
33)----------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.part
index ad65a4f08a..e038a7482d 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.part
@@ -60,14 +60,14 @@ logical_plan
02)--Projection: supplier.s_name, supplier.s_address
03)----LeftSemi Join: supplier.s_suppkey = __correlated_sq_2.ps_suppkey
04)------Projection: supplier.s_suppkey, supplier.s_name, supplier.s_address
-05)--------Inner Join: supplier.s_nationkey = nation.n_nationkey
+05)--------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey
06)----------TableScan: supplier projection=[s_suppkey, s_name, s_address, s_nationkey]
07)----------Projection: nation.n_nationkey
08)------------Filter: nation.n_name = Utf8View("CANADA")
09)--------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("CANADA")]
10)------SubqueryAlias: __correlated_sq_2
11)--------Projection: partsupp.ps_suppkey
-12)----------Inner Join: partsupp.ps_partkey = __scalar_sq_3.l_partkey, partsupp.ps_suppkey = __scalar_sq_3.l_suppkey Filter: CAST(partsupp.ps_availqty AS Float64) > __scalar_sq_3.Float64(0.5) * sum(lineitem.l_quantity)
+12)----------LeftSemi Join: partsupp.ps_partkey = __scalar_sq_3.l_partkey, partsupp.ps_suppkey = __scalar_sq_3.l_suppkey Filter: CAST(partsupp.ps_availqty AS Float64) > __scalar_sq_3.Float64(0.5) * sum(lineitem.l_quantity)
13)------------LeftSemi Join: partsupp.ps_partkey = __correlated_sq_1.p_partkey
14)--------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_availqty]
15)--------------SubqueryAlias: __correlated_sq_1
@@ -85,7 +85,7 @@ physical_plan
02)--SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true]
03)----HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_name@1, s_address@2]
04)------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=4
-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[s_suppkey@0, s_name@1, s_address@2]
+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@3, n_nationkey@0)], projection=[s_suppkey@0, s_name@1, s_address@2]
06)----------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=1
07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], constraints=[PrimaryKey([0])], file_type=csv, has_header=false
08)----------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4
@@ -93,7 +93,7 @@ physical_plan
10)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
11)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], constraints=[PrimaryKey([0])], file_type=csv, has_header=false
12)------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4
-13)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1]
+13)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1]
14)----------RepartitionExec: partitioning=Hash([ps_partkey@0, ps_suppkey@1], 4), input_partitions=4
15)------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ps_partkey@0, p_partkey@0)]
16)--------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part
index 2001aa8df0..812f5d2cba 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part
@@ -65,30 +65,29 @@ logical_plan
05)--------LeftAnti Join: l1.l_orderkey = __correlated_sq_2.l_orderkey Filter: __correlated_sq_2.l_suppkey != l1.l_suppkey
06)----------LeftSemi Join: l1.l_orderkey = __correlated_sq_1.l_orderkey Filter: __correlated_sq_1.l_suppkey != l1.l_suppkey
07)------------Projection: supplier.s_name, l1.l_orderkey, l1.l_suppkey
-08)--------------Inner Join: supplier.s_nationkey = nation.n_nationkey
-09)----------------Projection: supplier.s_name, supplier.s_nationkey, l1.l_orderkey, l1.l_suppkey
-10)------------------Inner Join: l1.l_orderkey = orders.o_orderkey
-11)--------------------Projection: supplier.s_name, supplier.s_nationkey, l1.l_orderkey, l1.l_suppkey
-12)----------------------Inner Join: supplier.s_suppkey = l1.l_suppkey
-13)------------------------TableScan: supplier projection=[s_suppkey, s_name, s_nationkey]
-14)------------------------SubqueryAlias: l1
-15)--------------------------Projection: lineitem.l_orderkey, lineitem.l_suppkey
-16)----------------------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate
-17)------------------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]
-18)--------------------Projection: orders.o_orderkey
-19)----------------------Filter: orders.o_orderstatus = Utf8View("F")
-20)------------------------TableScan: orders projection=[o_orderkey, o_orderstatus], partial_filters=[orders.o_orderstatus = Utf8View("F")]
-21)----------------Projection: nation.n_nationkey
-22)------------------Filter: nation.n_name = Utf8View("SAUDI ARABIA")
-23)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("SAUDI ARABIA")]
-24)------------SubqueryAlias: __correlated_sq_1
-25)--------------SubqueryAlias: l2
-26)----------------TableScan: lineitem projection=[l_orderkey, l_suppkey]
-27)----------SubqueryAlias: __correlated_sq_2
-28)------------SubqueryAlias: l3
-29)--------------Projection: lineitem.l_orderkey, lineitem.l_suppkey
-30)----------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate
-31)------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]
+08)--------------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey
+09)----------------LeftSemi Join: l1.l_orderkey = orders.o_orderkey
+10)------------------Projection: supplier.s_name, supplier.s_nationkey, l1.l_orderkey, l1.l_suppkey
+11)--------------------Inner Join: supplier.s_suppkey = l1.l_suppkey
+12)----------------------TableScan: supplier projection=[s_suppkey, s_name, s_nationkey]
+13)----------------------SubqueryAlias: l1
+14)------------------------Projection: lineitem.l_orderkey, lineitem.l_suppkey
+15)--------------------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate
+16)----------------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]
+17)------------------Projection: orders.o_orderkey
+18)--------------------Filter: orders.o_orderstatus = Utf8View("F")
+19)----------------------TableScan: orders projection=[o_orderkey, o_orderstatus], partial_filters=[orders.o_orderstatus = Utf8View("F")]
+20)----------------Projection: nation.n_nationkey
+21)------------------Filter: nation.n_name = Utf8View("SAUDI ARABIA")
+22)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("SAUDI ARABIA")]
+23)------------SubqueryAlias: __correlated_sq_1
+24)--------------SubqueryAlias: l2
+25)----------------TableScan: lineitem projection=[l_orderkey, l_suppkey]
+26)----------SubqueryAlias: __correlated_sq_2
+27)------------SubqueryAlias: l3
+28)--------------Projection: lineitem.l_orderkey, lineitem.l_suppkey
+29)----------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate
+30)------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]
physical_plan
01)SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST]
02)--SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true]
@@ -99,9 +98,9 @@ physical_plan
07)------------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0]
08)--------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0
09)----------------RepartitionExec: partitioning=Hash([l_orderkey@1], 4), input_partitions=4
-10)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@1, n_nationkey@0)], projection=[s_name@0, l_orderkey@2, l_suppkey@3]
+10)------------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@1, n_nationkey@0)], projection=[s_name@0, l_orderkey@2, l_suppkey@3]
11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@1], 4), input_partitions=4
-12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@2, o_orderkey@0)], projection=[s_name@0, s_nationkey@1, l_orderkey@2, l_suppkey@3]
+12)----------------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(l_orderkey@2, o_orderkey@0)]
13)------------------------RepartitionExec: partitioning=Hash([l_orderkey@2], 4), input_partitions=4
14)--------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4]
15)----------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part
index 7a3523b088..a92a752211 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part
@@ -48,7 +48,7 @@ logical_plan
04)------Projection: orders.o_orderdate, orders.o_shippriority, lineitem.l_orderkey, lineitem.l_extendedprice, lineitem.l_discount
05)--------Inner Join: orders.o_orderkey = lineitem.l_orderkey
06)----------Projection: orders.o_orderkey, orders.o_orderdate, orders.o_shippriority
-07)------------Inner Join: customer.c_custkey = orders.o_custkey
+07)------------RightSemi Join: customer.c_custkey = orders.o_custkey
08)--------------Projection: customer.c_custkey
09)----------------Filter: customer.c_mktsegment = Utf8View("BUILDING")
10)------------------TableScan: customer projection=[c_custkey, c_mktsegment], partial_filters=[customer.c_mktsegment = Utf8View("BUILDING")]
@@ -64,7 +64,7 @@ physical_plan
04)------AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5]
06)----------RepartitionExec: partitioning=Hash([o_orderkey@0], 4), input_partitions=4
-07)------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4]
+07)------------HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@0, o_orderdate@2, o_shippriority@3]
08)--------------RepartitionExec: partitioning=Hash([c_custkey@0], 4), input_partitions=4
09)----------------FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0]
10)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:0..606529], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:606529..1213058], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:1213058..1819587], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:1819587..2426114]]}, projection=[c_custkey, c_mktsegment], constraints=[PrimaryKey([0])], file_type=csv, has_header=false
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part
index 6dd06b269e..036c0e3b8c 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part
@@ -47,7 +47,7 @@ logical_plan
02)--Projection: nation.n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS revenue
03)----Aggregate: groupBy=[[nation.n_name]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(1,20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]
04)------Projection: lineitem.l_extendedprice, lineitem.l_discount, nation.n_name
-05)--------Inner Join: nation.n_regionkey = region.r_regionkey
+05)--------LeftSemi Join: nation.n_regionkey = region.r_regionkey
06)----------Projection: lineitem.l_extendedprice, lineitem.l_discount, nation.n_name, nation.n_regionkey
07)------------Inner Join: supplier.s_nationkey = nation.n_nationkey
08)--------------Projection: lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey
@@ -73,7 +73,7 @@ physical_plan
04)------AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
05)--------RepartitionExec: partitioning=Hash([n_name@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
-07)------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)], projection=[l_extendedprice@0, l_discount@1, n_name@2]
+07)------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@3, r_regionkey@0)], projection=[l_extendedprice@0, l_discount@1, n_name@2]
08)--------------RepartitionExec: partitioning=Hash([n_regionkey@3], 4), input_partitions=4
09)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[l_extendedprice@0, l_discount@1, n_name@4, n_regionkey@5]
10)------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part
index c38930cb5b..902413e9ef 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part
@@ -61,7 +61,7 @@ logical_plan
03)----Aggregate: groupBy=[[all_nations.o_year]], aggr=[[sum(CASE WHEN all_nations.nation = Utf8View("BRAZIL") THEN all_nations.volume ELSE Decimal128(0.0000,38,4) END) AS sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]]
04)------SubqueryAlias: all_nations
05)--------Projection: date_part(Utf8("YEAR"), orders.o_orderdate) AS o_year, lineitem.l_extendedprice * (Decimal128(1,20,0) - lineitem.l_discount) AS volume, n2.n_name AS nation
-06)----------Inner Join: n1.n_regionkey = region.r_regionkey
+06)----------LeftSemi Join: n1.n_regionkey = region.r_regionkey
07)------------Projection: lineitem.l_extendedprice, lineitem.l_discount, orders.o_orderdate, n1.n_regionkey, n2.n_name
08)--------------Inner Join: supplier.s_nationkey = n2.n_nationkey
09)----------------Projection: lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey, orders.o_orderdate, n1.n_regionkey
@@ -73,7 +73,7 @@ logical_plan
15)----------------------------Projection: lineitem.l_orderkey, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey
16)------------------------------Inner Join: lineitem.l_suppkey = supplier.s_suppkey
17)--------------------------------Projection: lineitem.l_orderkey, lineitem.l_suppkey, lineitem.l_extendedprice, lineitem.l_discount
-18)----------------------------------Inner Join: part.p_partkey = lineitem.l_partkey
+18)----------------------------------RightSemi Join: part.p_partkey = lineitem.l_partkey
19)------------------------------------Projection: part.p_partkey
20)--------------------------------------Filter: part.p_type = Utf8View("ECONOMY ANODIZED STEEL")
21)----------------------------------------TableScan: part projection=[p_partkey, p_type], partial_filters=[part.p_type = Utf8View("ECONOMY ANODIZED STEEL")]
@@ -97,7 +97,7 @@ physical_plan
05)--------RepartitionExec: partitioning=Hash([o_year@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE 0.0000 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]
07)------------ProjectionExec: expr=[date_part(YEAR, o_orderdate@0) as o_year, l_extendedprice@1 * (1 - l_discount@2) as volume, n_name@3 as nation]
-08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)], projection=[o_orderdate@2, l_extendedprice@0, l_discount@1, n_name@4]
+08)--------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@3, r_regionkey@0)], projection=[o_orderdate@2, l_extendedprice@0, l_discount@1, n_name@4]
09)----------------RepartitionExec: partitioning=Hash([n_regionkey@3], 4), input_partitions=4
10)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[l_extendedprice@0, l_discount@1, o_orderdate@3, n_regionkey@4, n_name@6]
11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4
@@ -109,7 +109,7 @@ physical_plan
17)--------------------------------RepartitionExec: partitioning=Hash([l_orderkey@0], 4), input_partitions=4
18)----------------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@1, s_suppkey@0)], projection=[l_orderkey@0, l_extendedprice@2, l_discount@3, s_nationkey@5]
19)------------------------------------RepartitionExec: partitioning=Hash([l_suppkey@1], 4), input_partitions=4
-20)--------------------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5]
+20)--------------------------------------HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@0, l_suppkey@2, l_extendedprice@3, l_discount@4]
21)----------------------------------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4
22)------------------------------------------FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0]
23)--------------------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:0..597773], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:597773..1195546], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1195546..1793319], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1793319..2391090]]}, projection=[p_partkey, p_type], constraints=[PrimaryKey([0])], file_type=csv, has_header=false
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part
index ca09252a4b..ade7ed7a6c 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part
@@ -64,16 +64,15 @@ logical_plan
10)------------------Inner Join: lineitem.l_suppkey = partsupp.ps_suppkey, lineitem.l_partkey = partsupp.ps_partkey
11)--------------------Projection: lineitem.l_orderkey, lineitem.l_partkey, lineitem.l_suppkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey
12)----------------------Inner Join: lineitem.l_suppkey = supplier.s_suppkey
-13)------------------------Projection: lineitem.l_orderkey, lineitem.l_partkey, lineitem.l_suppkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount
-14)--------------------------Inner Join: part.p_partkey = lineitem.l_partkey
-15)----------------------------Projection: part.p_partkey
-16)------------------------------Filter: part.p_name LIKE Utf8View("%green%")
-17)--------------------------------TableScan: part projection=[p_partkey, p_name], partial_filters=[part.p_name LIKE Utf8View("%green%")]
-18)----------------------------TableScan: lineitem projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount]
-19)------------------------TableScan: supplier projection=[s_suppkey, s_nationkey]
-20)--------------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]
-21)----------------TableScan: orders projection=[o_orderkey, o_orderdate]
-22)------------TableScan: nation projection=[n_nationkey, n_name]
+13)------------------------RightSemi Join: part.p_partkey = lineitem.l_partkey
+14)--------------------------Projection: part.p_partkey
+15)----------------------------Filter: part.p_name LIKE Utf8View("%green%")
+16)------------------------------TableScan: part projection=[p_partkey, p_name], partial_filters=[part.p_name LIKE Utf8View("%green%")]
+17)--------------------------TableScan: lineitem projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount]
+18)------------------------TableScan: supplier projection=[s_suppkey, s_nationkey]
+19)--------------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]
+20)----------------TableScan: orders projection=[o_orderkey, o_orderdate]
+21)------------TableScan: nation projection=[n_nationkey, n_name]
physical_plan
01)SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC], fetch=10
02)--SortExec: TopK(fetch=10), expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true]
@@ -90,7 +89,7 @@ physical_plan
13)------------------------RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 4), input_partitions=4
14)--------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, s_suppkey@0)], projection=[l_orderkey@0, l_partkey@1, l_suppkey@2, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@7]
15)----------------------------RepartitionExec: partitioning=Hash([l_suppkey@2], 4), input_partitions=4
-16)------------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6]
+16)------------------------------HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(p_partkey@0, l_partkey@1)]
17)--------------------------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4
18)----------------------------------FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0]
19)------------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:0..597773], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:597773..1195546], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1195546..1793319], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1793319..2391090]]}, projection=[p_partkey, p_name], constraints=[PrimaryKey([0])], file_type=csv, has_header=false