Apache DataFusion SQL Query Engine
perf: Convert inner joins to semi joins when equivalent (#22652)
Neil Conwayb8998c762bb8
diff --git a/datafusion/optimizer/src/eliminate_join.rs b/datafusion/optimizer/src/eliminate_join.rsindex 885910c1e4..cce17c07b5 100644--- a/datafusion/optimizer/src/eliminate_join.rs+++ b/datafusion/optimizer/src/eliminate_join.rs@@ -15,19 +15,135 @@// specific language governing permissions and limitations// under the License.-//! [`EliminateJoin`] rewrites `INNER JOIN` with `true`/`null`-use crate::optimizer::ApplyOrder;+//! [`EliminateJoin`] rewrites inner joins to simpler forms to make them cheaper+//! to evaluate. We implement two distinct rewrites:+//!+//! * An inner join can be rewritten to an empty relation if the join condition+//! is trivially false.+//!+//! * An inner join `L ⋈ R` can be rewritten to a left semi join `L ⋉ R`+//! (`LeftSemi`), which keeps the rows of L that have a match in R and outputs+//! only L's columns. The rewrite to `L ⋉ R` is valid when both of the+//! following are true:+//!+//! 1. None of R's columns are referenced above the join.+//! 2. R does not observably multiply L's rows. This holds when either the+//! join's ancestors are duplicate-insensitive (e.g., DISTINCT) or we can use+//! functional dependencies to prove that each L row matches at most one R+//! row (R is provably unique on the join keys).+//!+//! # Overview+//!+//! `rewrite_subtree` walks the plan top-down, threading two pieces of context+//! down to each join:+//!+//! * `live` — which of the join's output columns are referenced above it. It is+//! propagated top-down: each node asks its children only for the columns it+//! needs from them, so a projection or aggregate asks for just the columns its+//! expressions reference, dropping the rest (the narrowing); a join splits the+//! set across its two inputs.+//! * `duplicate_insensitive` — whether emitting each row once instead of many+//! times will not change the output. A duplicate-collapsing node (e.g.,+//! DISTINCT, GROUP BY with no aggregate functions, or the existence side of a+//! semi/anti/mark join) sets it `true` for its subtree, and it propagates+//! downward until a node that makes the row count observable again (a `LIMIT`,+//! a top-N sort, ...) clears it. It is therefore fixed by the nearest such+//! node, not by the whole ancestor chain: a collapsing node shields its subtree,+//! so a duplicate-sensitive node further above does not matter.+//!+//! At each join, `rewritten_join_type` combines this context with the side's+//! functional dependencies to choose `Inner`, `LeftSemi`, or `RightSemi`. Most+//! node types just forward the context to their single child via+//! `rewrite_single_input`; nodes that alter column requirements or+//! duplicate-sensitivity (projection, aggregate, sort, ...) adjust it first.+use crate::utils::for_each_referenced_index;use crate::{OptimizerConfig, OptimizerRule};-use datafusion_common::tree_node::Transformed;-use datafusion_common::{Result, ScalarValue};-use datafusion_expr::JoinType::Inner;+use datafusion_common::tree_node::{Transformed, TreeNode};+use datafusion_common::{+ DFSchema, Dependency, HashSet, NullEquality, Result, ScalarValue,+};use datafusion_expr::{- Expr,- logical_plan::{EmptyRelation, LogicalPlan},+ Expr, JoinType,+ logical_plan::{+ Aggregate, Distinct, DistinctOn, EmptyRelation, Filter, Join, Limit, LogicalPlan,+ Partitioning, Projection, Repartition, Sort, SubqueryAlias,+ },};+use std::sync::Arc;++/// The columns that are "live" at a plan node, i.e., which of its output+/// columns are referenced by an ancestor node. Represented as a set of column+/// indices, relative to the node's schema.+///+/// See the module-level docs for how this set is threaded down the plan and+/// narrowed or split at each node.+#[derive(Debug, Default, Clone)]+struct LiveColumns(HashSet<usize>);++impl LiveColumns {+ fn new() -> Self {+ Self(HashSet::new())+ }++ /// Every column of `schema` is live.+ fn all(schema: &DFSchema) -> Self {+ Self((0..schema.fields().len()).collect())+ }++ /// The columns of `schema` referenced by any of `exprs`.+ fn try_new<'a>(+ exprs: impl IntoIterator<Item = &'a Expr>,+ schema: &DFSchema,+ ) -> Result<Self> {+ let mut live = Self::new();+ live.extend_from(exprs, schema)?;+ Ok(live)+ }++ /// Inserts the index, within `schema`, of every column referenced by any of+ /// `exprs`, including columns reached through correlated subquery outer+ /// references.+ fn extend_from<'a>(+ &mut self,+ exprs: impl IntoIterator<Item = &'a Expr>,+ schema: &DFSchema,+ ) -> Result<()> {+ for expr in exprs {+ for_each_referenced_index(expr, schema, |idx| {+ self.0.insert(idx);+ })?;+ }+ Ok(())+ }++ fn insert(&mut self, idx: usize) {+ self.0.insert(idx);+ }++ fn is_empty(&self) -> bool {+ self.0.is_empty()+ }-/// Eliminates joins when join condition is false.-/// Replaces joins when inner join condition is true with a cross join.+ /// Splits live columns spanning a join's combined output (the left input's+ /// columns first, then the right input's) into the per-side sets, rebasing+ /// the right side's indices to start at zero. `left_len` is the number of+ /// columns contributed by the left input.+ fn split_at(&self, left_len: usize) -> (Self, Self) {+ let mut left = Self::new();+ let mut right = Self::new();+ for &idx in &self.0 {+ if idx < left_len {+ left.insert(idx);+ } else {+ right.insert(idx - left_len);+ }+ }+ (left, right)+ }+}++/// Rewrites an inner join to a semi join when one input only filters the other,+/// and replaces an always-false inner join with an empty relation.#[derive(Default, Debug)]pub struct EliminateJoin;@@ -42,44 +158,438 @@ impl OptimizerRule for EliminateJoin {"eliminate_join"}- fn apply_order(&self) -> Option<ApplyOrder> {- Some(ApplyOrder::TopDown)- }-fn rewrite(&self,plan: LogicalPlan,_config: &dyn OptimizerConfig,) -> Result<Transformed<LogicalPlan>> {- match plan {- LogicalPlan::Join(join) if join.join_type == Inner && join.on.is_empty() => {- match join.filter {- Some(Expr::Literal(ScalarValue::Boolean(Some(false)), _)) => Ok(- Transformed::yes(LogicalPlan::EmptyRelation(EmptyRelation {- produce_one_row: false,- schema: join.schema,- })),- ),- _ => Ok(Transformed::no(LogicalPlan::Join(join))),+ let live = LiveColumns::all(plan.schema());+ rewrite_subtree(plan, live, false)+ }+}++/// Rewrites `plan` and everything below it, including joins nested inside+/// subquery expressions.+///+/// [`rewrite_node`] handles the node itself and recurses into its plan+/// children; this wrapper additionally descends into the node's own subquery+/// expressions. Each subquery is seeded as a fresh root, since its columns are+/// independent of the enclosing plan's `live` set.+fn rewrite_subtree(+ plan: LogicalPlan,+ live: LiveColumns,+ duplicate_insensitive: bool,+) -> Result<Transformed<LogicalPlan>> {+ rewrite_node(plan, live, duplicate_insensitive)?.transform_data(|plan| {+ plan.map_subqueries(|subquery| {+ let live = LiveColumns::all(subquery.schema());+ rewrite_subtree(subquery, live, false)+ })+ })+}++fn rewrite_node(+ plan: LogicalPlan,+ live: LiveColumns,+ duplicate_insensitive: bool,+) -> Result<Transformed<LogicalPlan>> {+ match plan {+ // The only arm that rewrites a join; the rest just thread context down to one.+ LogicalPlan::Join(join) => rewrite_join(join, &live, duplicate_insensitive),+ LogicalPlan::Projection(Projection {+ expr,+ input,+ schema,+ ..+ }) => {+ // Narrows `live` to the columns the projection's expressions reference.+ let child_live = LiveColumns::try_new(&expr, input.schema())?;+ rewrite_single_input(input, child_live, duplicate_insensitive, |input| {+ Ok(LogicalPlan::Projection(Projection::try_new_with_schema(+ expr, input, schema,+ )?))+ })+ }+ LogicalPlan::Filter(Filter {+ predicate, input, ..+ }) => {+ // Adds the predicate's columns to `live` (a side used only by the filter stays live).+ let mut child_live = live;+ child_live.extend_from([&predicate], input.schema())?;+ rewrite_single_input(input, child_live, duplicate_insensitive, |input| {+ Ok(LogicalPlan::Filter(Filter::new(predicate, input)))+ })+ }+ LogicalPlan::Aggregate(Aggregate {+ input,+ group_expr,+ aggr_expr,+ schema,+ ..+ }) => {+ // Narrows `live` to the grouping and aggregate expressions' columns.+ let child_live = LiveColumns::try_new(+ group_expr.iter().chain(&aggr_expr),+ input.schema(),+ )?;++ // A grouping aggregate with no aggregate functions (`GROUP BY` with+ // an empty `aggr_expr`) only observes which group-key values exist,+ // not how many rows produced them, so its input is duplicate-+ // insensitive.+ let child_duplicate_insensitive =+ !group_expr.is_empty() && aggr_expr.is_empty();++ rewrite_single_input(+ input,+ child_live,+ child_duplicate_insensitive,+ |input| {+ Ok(LogicalPlan::Aggregate(Aggregate::try_new_with_schema(+ input, group_expr, aggr_expr, schema,+ )?))+ },+ )+ }+ LogicalPlan::Distinct(Distinct::All(input)) => {+ // `SELECT DISTINCT *` is equivalent to a no-aggregate `GROUP BY`+ // over every input column, so the input is duplicate-insensitive,+ // but every column is part of the dedup key.+ let child_live = LiveColumns::all(input.schema());+ rewrite_single_input(input, child_live, true, |input| {+ Ok(LogicalPlan::Distinct(Distinct::All(input)))+ })+ }+ LogicalPlan::Distinct(Distinct::On(DistinctOn {+ on_expr,+ select_expr,+ sort_expr,+ input,+ schema,+ })) => {+ // `DISTINCT ON (on) select [ORDER BY sort]` is a no-aggregate+ // `GROUP BY` on the columns it reads, so its input is duplicate-+ // insensitive; the live columns are exactly those of the+ // ON/SELECT/ORDER BY expressions.+ let mut child_live =+ LiveColumns::try_new(on_expr.iter().chain(&select_expr), input.schema())?;+ if let Some(sort_expr) = &sort_expr {+ child_live+ .extend_from(sort_expr.iter().map(|s| &s.expr), input.schema())?;+ }++ rewrite_single_input(input, child_live, true, |input| {+ Ok(LogicalPlan::Distinct(Distinct::On(DistinctOn {+ on_expr,+ select_expr,+ sort_expr,+ input,+ schema,+ })))+ })+ }+ LogicalPlan::Sort(Sort { expr, input, fetch }) => {+ // Adds the sort-key columns to `live`.+ let mut child_live = live;+ child_live.extend_from(expr.iter().map(|s| &s.expr), input.schema())?;++ // A `fetch` (top-N) makes the row count observable, so duplicate-+ // insensitivity does not survive past it.+ let child_duplicate_insensitive = duplicate_insensitive && fetch.is_none();+ rewrite_single_input(+ input,+ child_live,+ child_duplicate_insensitive,+ |input| Ok(LogicalPlan::Sort(Sort { expr, input, fetch })),+ )+ }+ LogicalPlan::Limit(Limit { skip, fetch, input }) => {+ // LIMIT makes the row count observable, so it clears duplicate-insensitivity.+ rewrite_single_input(input, live, false, |input| {+ Ok(LogicalPlan::Limit(Limit { skip, fetch, input }))+ })+ }+ LogicalPlan::SubqueryAlias(SubqueryAlias { input, alias, .. }) => {+ // Re-aliases columns 1:1, so `live` and duplicate-sensitivity pass through unchanged.+ rewrite_single_input(input, live, duplicate_insensitive, |input| {+ Ok(LogicalPlan::SubqueryAlias(SubqueryAlias::try_new(+ input, alias,+ )?))+ })+ }+ LogicalPlan::Repartition(Repartition {+ input,+ partitioning_scheme,+ }) => {+ // Adds any partitioning-key columns to `live`; duplicate-sensitivity is unchanged.+ let mut child_live = live;+ match &partitioning_scheme {+ Partitioning::Hash(exprs, _) | Partitioning::DistributeBy(exprs) => {+ child_live.extend_from(exprs, input.schema())?;}+ Partitioning::Range(range) => {+ child_live.extend_from(+ range.ordering().iter().map(|sort_expr| &sort_expr.expr),+ input.schema(),+ )?;+ }+ Partitioning::RoundRobinBatch(_) => {}}- _ => Ok(Transformed::no(plan)),+ rewrite_single_input(input, child_live, duplicate_insensitive, |input| {+ Ok(LogicalPlan::Repartition(Repartition {+ input,+ partitioning_scheme,+ }))+ })}+ // Conservatively treat any other plan node as a fresh root, since we are+ // not sure of its semantics with respect to duplicates or live columns.+ _ => plan.map_children(|child| {+ let live = LiveColumns::all(child.schema());+ rewrite_subtree(child, live, false)+ }),}+}- fn supports_rewrite(&self) -> bool {- true+/// Recurses into a single-input node's child, threading `child_live` and+/// `duplicate_insensitive` down, then rebuilds the node from the (possibly+/// rewritten) child via `rebuild`. The child's `Transformed` flag is preserved,+/// so the node is reported as changed exactly when its child changed.+fn rewrite_single_input<F>(+ input: Arc<LogicalPlan>,+ child_live: LiveColumns,+ duplicate_insensitive: bool,+ rebuild: F,+) -> Result<Transformed<LogicalPlan>>+where+ F: FnOnce(Arc<LogicalPlan>) -> Result<LogicalPlan>,+{+ rewrite_subtree(+ Arc::unwrap_or_clone(input),+ child_live,+ duplicate_insensitive,+ )?+ .map_data(|input| rebuild(Arc::new(input)))+}++fn rewrite_join(+ join: Join,+ live: &LiveColumns,+ duplicate_insensitive: bool,+) -> Result<Transformed<LogicalPlan>> {+ if join.join_type == JoinType::Inner+ && join.on.is_empty()+ && matches!(+ join.filter.as_ref(),+ Some(Expr::Literal(ScalarValue::Boolean(Some(false)), _))+ )+ {+ return Ok(Transformed::yes(LogicalPlan::EmptyRelation(+ EmptyRelation {+ produce_one_row: false,+ schema: join.schema,+ },+ )));+ }++ let (visible_left, visible_right) = split_join_output_columns(&join, live);++ let rewritten_join_type =+ rewritten_join_type(&join, &visible_left, &visible_right, duplicate_insensitive);++ let (mut left_live, mut right_live) = match rewritten_join_type {+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {+ (visible_left, LiveColumns::new())+ }+ JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {+ (LiveColumns::new(), visible_right)+ }+ _ => (visible_left, visible_right),+ };++ add_join_condition_columns(&join, &mut left_live, &mut right_live)?;++ let (left_dup_insensitive, right_dup_insensitive) =+ child_duplicate_insensitivity(rewritten_join_type, duplicate_insensitive);++ let left = rewrite_subtree(+ Arc::unwrap_or_clone(join.left),+ left_live,+ left_dup_insensitive,+ )?;+ let right = rewrite_subtree(+ Arc::unwrap_or_clone(join.right),+ right_live,+ right_dup_insensitive,+ )?;++ let changed =+ left.transformed || right.transformed || rewritten_join_type != join.join_type;+ let left = Arc::new(left.data);+ let right = Arc::new(right.data);++ if changed {+ // The join type or an input changed, so the output schema may have+ // narrowed; recompute it via `try_new`.+ Ok(Transformed::yes(LogicalPlan::Join(Join::try_new(+ left,+ right,+ join.on,+ join.filter,+ rewritten_join_type,+ join.join_constraint,+ join.null_equality,+ join.null_aware,+ )?)))+ } else {+ // Nothing changed; reassemble the join reusing its existing schema rather+ // than recomputing it.+ Ok(Transformed::no(LogicalPlan::Join(Join {+ left,+ right,+ on: join.on,+ filter: join.filter,+ join_type: join.join_type,+ join_constraint: join.join_constraint,+ schema: join.schema,+ null_equality: join.null_equality,+ null_aware: join.null_aware,+ })))}}+/// Returns which join inputs can safely ignore duplicate rows from their own+/// descendants. For semi/anti/mark joins, duplicates from the existence side do+/// not change the result even when the parent itself is duplicate-sensitive.+fn child_duplicate_insensitivity(+ join_type: JoinType,+ duplicate_insensitive: bool,+) -> (bool, bool) {+ match join_type {+ JoinType::Inner => (duplicate_insensitive, duplicate_insensitive),+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {+ (duplicate_insensitive, true)+ }+ JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {+ (true, duplicate_insensitive)+ }+ JoinType::Left | JoinType::Right | JoinType::Full => (false, false),+ }+}++/// Rewrites an inner join to a semi join when the removed side has no+/// parent-visible columns and either the parent ignores duplicate output rows or+/// the removed side is unique on the join keys.+fn rewritten_join_type(+ join: &Join,+ visible_left: &LiveColumns,+ visible_right: &LiveColumns,+ duplicate_insensitive: bool,+) -> JoinType {+ if join.join_type != JoinType::Inner || join.on.is_empty() {+ return join.join_type;+ }++ let can_remove_right = duplicate_insensitive+ || side_unique_on_join(+ join.right.schema(),+ join.on.iter().map(|(_, right)| right),+ join.null_equality,+ );+ if visible_right.is_empty() && can_remove_right {+ return JoinType::LeftSemi;+ }++ let can_remove_left = duplicate_insensitive+ || side_unique_on_join(+ join.left.schema(),+ join.on.iter().map(|(left, _)| left),+ join.null_equality,+ );+ if visible_left.is_empty() && can_remove_left {+ return JoinType::RightSemi;+ }++ JoinType::Inner+}++fn add_join_condition_columns(+ join: &Join,+ left_live: &mut LiveColumns,+ right_live: &mut LiveColumns,+) -> Result<()> {+ left_live.extend_from(join.on.iter().map(|(l, _)| l), join.left.schema())?;+ right_live.extend_from(join.on.iter().map(|(_, r)| r), join.right.schema())?;++ if let Some(filter) = &join.filter {+ left_live.extend_from([filter], join.left.schema())?;+ right_live.extend_from([filter], join.right.schema())?;+ }++ Ok(())+}++fn split_join_output_columns(+ join: &Join,+ live: &LiveColumns,+) -> (LiveColumns, LiveColumns) {+ let left_len = join.left.schema().fields().len();+ match join.join_type {+ JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full => {+ live.split_at(left_len)+ }+ // A semi/anti/mark join outputs only the surviving side's columns, with+ // the same index space, so `live` passes straight through to that side.+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {+ (live.clone(), LiveColumns::new())+ }+ JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {+ (LiveColumns::new(), live.clone())+ }+ }+}++fn side_unique_on_join<'a>(+ schema: &DFSchema,+ join_exprs: impl Iterator<Item = &'a Expr>,+ null_equality: NullEquality,+) -> bool {+ let join_key_indices = join_exprs+ .filter_map(|expr| match expr {+ Expr::Alias(alias) => alias.expr.as_ref().try_as_col(),+ _ => expr.try_as_col(),+ })+ .filter_map(|column| schema.maybe_index_of_column(column))+ .collect::<Vec<usize>>();++ schema.functional_dependencies().iter().any(|dependency| {+ dependency.mode == Dependency::Single+ && (!dependency.nullable || null_equality == NullEquality::NullEqualsNothing)+ && dependency+ .source_indices+ .iter()+ .all(|idx| join_key_indices.contains(idx))+ })+}+#[cfg(test)]mod tests {use crate::OptimizerContext;use crate::assert_optimized_plan_eq_snapshot;use crate::eliminate_join::EliminateJoin;- use datafusion_common::Result;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::{+ Constraint, Constraints, NullEquality, Result, ScalarValue, SplitPoint,+ };use datafusion_expr::JoinType::Inner;- use datafusion_expr::{lit, logical_plan::builder::LogicalPlanBuilder};+ use datafusion_expr::{+ Expr, JoinType, Partitioning, RangePartitioning, col, exists, lit,+ logical_plan::builder::{+ LogicalPlanBuilder, table_scan, table_source_with_constraints,+ },+ out_ref_col,+ };+ use datafusion_functions_aggregate::expr_fn::count;use std::sync::Arc;macro_rules! assert_optimized_plan_equal {@@ -110,4 +620,591 @@ mod tests {assert_optimized_plan_equal!(plan, @"EmptyRelation: rows=0")}++ #[test]+ fn inner_to_left_semi_when_removed_side_is_unique() -> Result<()> {+ let plan = left_join_right_with_constraints(primary_key_on_id())?+ .project(vec![col("l.x")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.x+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn inner_to_left_semi_when_removed_side_is_unique_with_join_filter() -> Result<()> {+ let right = scan("r", &test_schema(), primary_key_on_id())?;+ let plan =+ LogicalPlanBuilder::from(scan("l", &test_schema(), Constraints::default())?)+ .join(+ right,+ Inner,+ (vec!["l.id"], vec!["r.id"]),+ Some(col("r.y").gt(col("l.x"))),+ )?+ .project(vec![col("l.x")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.x+ LeftSemi Join: l.id = r.id Filter: r.y > l.x+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn inner_to_right_semi_when_removed_side_is_unique() -> Result<()> {+ let plan = left_with_constraints_join_right(primary_key_on_id())?+ .project(vec![col("r.y")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: r.y+ RightSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn inner_to_left_semi_for_duplicate_insensitive_parent() -> Result<()> {+ let plan = left_join_right()?+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[]]+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn aggregate_with_aggregates_is_not_duplicate_insensitive() -> Result<()> {+ // A `GROUP BY` *with* aggregate functions observes how many rows fall in+ // each group, so its input is not duplicate-insensitive. With a non-unique+ // right side the join must stay an inner join: collapsing it to a semi+ // join would drop matching duplicates and undercount `count(l.id)`.+ let plan = left_join_right()?+ .aggregate(vec![col("l.x")], vec![count(col("l.id"))])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[count(l.id)]]+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn duplicate_insensitive_context_propagates_through_join_tree() -> Result<()> {+ let left = scan("l", &test_schema(), Constraints::default())?;+ let middle = scan("m", &test_schema(), Constraints::default())?;+ let right = scan("r", &test_schema(), Constraints::default())?;++ let left_join_middle = LogicalPlanBuilder::from(left)+ .join(middle, Inner, (vec!["l.id"], vec!["m.id"]), None)?+ .build()?;++ let plan = LogicalPlanBuilder::from(left_join_middle)+ .join(right, Inner, (vec!["l.id"], vec!["r.id"]), None)?+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[]]+ LeftSemi Join: l.id = r.id+ LeftSemi Join: l.id = m.id+ TableScan: l+ TableScan: m+ TableScan: r+ ")+ }++ #[test]+ fn projection_does_not_rewrite_without_uniqueness() -> Result<()> {+ let plan = left_join_right()?.project(vec![col("l.x")])?.build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.x+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn required_filter_column_prevents_duplicate_insensitive_rewrite() -> Result<()> {+ let plan = left_join_right()?+ .filter(col("r.y").gt(lit(10_i32)))?+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[]]+ Filter: r.y > Int32(10)+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn distinct_star_keeps_unreferenced_side() -> Result<()> {+ // `SELECT DISTINCT *` deduplicates on every join-output column, including+ // the right side's. With a non-unique right side the inner join can+ // multiply left rows into distinct `(l, r)` combinations, so the join+ // must not be rewritten to a semi join (which would drop the right+ // columns from the DISTINCT key and undercount the result). This holds+ // even when the right side is unique on the join keys: its columns are+ // part of the DISTINCT key regardless.+ let plan = left_join_right()?+ .distinct()?+ .project(vec![col("l.x")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.x+ Distinct:+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn distinct_drops_unreferenced_side_when_projected() -> Result<()> {+ // `SELECT DISTINCT l.x` projects the right side away below the DISTINCT,+ // leaving it outside the dedup key. Like a no-aggregate `GROUP BY l.x`,+ // the DISTINCT makes the input duplicate-insensitive, so the inner join+ // collapses to a semi join even though the right side is not unique.+ let plan = left_join_right()?+ .project(vec![col("l.x")])?+ .distinct()?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Distinct:+ Projection: l.x+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn correlated_subquery_outer_ref_prevents_rewrite() -> Result<()> {+ // The aggregate makes the parent duplicate-insensitive, so absent any+ // other use of the right side the join would collapse to a semi join.+ // But the `EXISTS` subquery correlates on `r.y`, so the right side is+ // still needed and the join must stay an inner join. Otherwise the+ // semi join would drop `r`, orphaning the correlated `r.y` reference.+ let subquery =+ LogicalPlanBuilder::from(scan("s", &test_schema(), Constraints::default())?)+ .filter(col("s.id").eq(out_ref_col(DataType::Int32, "r.y")))?+ .project(vec![lit(1)])?+ .build()?;++ let plan = left_join_right()?+ .filter(exists(Arc::new(subquery)))?+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[]]+ Filter: EXISTS (<subquery>)+ Subquery:+ Projection: Int32(1)+ Filter: s.id = outer_ref(r.y)+ TableScan: s+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn inner_to_semi_inside_uncorrelated_subquery() -> Result<()> {+ // A join nested inside a (not-yet-decorrelated) subquery is still+ // rewritten, because `rewrite_subtree` descends into subquery plans+ // itself via `map_subqueries`. Here the subquery's projection keeps+ // only `l.x` and the removed side `r` is unique (PK), so the inner join+ // collapses to a semi join.+ let subquery = left_join_right_with_constraints(primary_key_on_id())?+ .project(vec![col("l.x")])?+ .build()?;++ let plan = LogicalPlanBuilder::from(scan(+ "outer",+ &test_schema(),+ Constraints::default(),+ )?)+ .filter(exists(Arc::new(subquery)))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: EXISTS (<subquery>)+ Subquery:+ Projection: l.x+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ TableScan: outer+ ")+ }++ #[test]+ fn inner_to_semi_inside_correlated_subquery() -> Result<()> {+ // `map_subqueries` descends into correlated subqueries too, not just+ // uncorrelated ones, so a join inside one is still rewritten. The+ // subquery correlates on `outer.id` (via the filter), but that reference+ // and the projection touch only `l`; `r` is unique (PK) and unreferenced,+ // so the inner join inside the subquery collapses to a semi join.+ let subquery = left_join_right_with_constraints(primary_key_on_id())?+ .filter(col("l.x").eq(out_ref_col(DataType::Int32, "outer.id")))?+ .project(vec![col("l.x")])?+ .build()?;++ let plan = LogicalPlanBuilder::from(scan(+ "outer",+ &test_schema(),+ Constraints::default(),+ )?)+ .filter(exists(Arc::new(subquery)))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: EXISTS (<subquery>)+ Subquery:+ Projection: l.x+ Filter: l.x = outer_ref(outer.id)+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ TableScan: outer+ ")+ }++ #[test]+ fn nullable_unique_rewrites_under_null_equals_nothing() -> Result<()> {+ // A `UNIQUE` (rather than `PRIMARY KEY`) constraint marks the key as+ // nullable. Under the default `NullEqualsNothing` join semantics a null+ // key matches nothing, so a unique side still yields at most one match+ // per left row and the inner join can become a semi join.+ let left = scan("l", &test_schema(), Constraints::default())?;+ let right = scan("r", &test_schema(), unique_on_x())?;+ let plan = LogicalPlanBuilder::from(left)+ .join(right, Inner, (vec!["l.x"], vec!["r.x"]), None)?+ .project(vec![col("l.id")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.id+ LeftSemi Join: l.x = r.x+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn nullable_unique_does_not_rewrite_under_null_equals_null() -> Result<()> {+ // With `NullEqualsNull` semantics two null keys compare equal, so a+ // nullable `UNIQUE` key no longer guarantees at most one match per left+ // row: several null-keyed right rows could match a null-keyed left row.+ // Uniqueness on the join keys is therefore not established and the inner+ // join must be preserved.+ let left = scan("l", &test_schema(), Constraints::default())?;+ let right = scan("r", &test_schema(), unique_on_x())?;+ let plan = LogicalPlanBuilder::from(left)+ .join_detailed(+ right,+ Inner,+ (vec!["l.x"], vec!["r.x"]),+ None,+ NullEquality::NullEqualsNull,+ )?+ .project(vec![col("l.id")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.id+ Inner Join: l.x = r.x+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn composite_unique_rewrites_when_join_covers_all_key_columns() -> Result<()> {+ // The removed side is unique on the composite key `(id, x)`. The join+ // equates both key columns, so each left row matches at most one right+ // row and the inner join can become a semi join.+ let left = scan("l", &test_schema(), Constraints::default())?;+ let right = scan("r", &test_schema(), composite_primary_key_on_id_x())?;+ let plan = LogicalPlanBuilder::from(left)+ .join(+ right,+ Inner,+ (vec!["l.id", "l.x"], vec!["r.id", "r.x"]),+ None,+ )?+ .project(vec![col("l.y")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.y+ LeftSemi Join: l.id = r.id, l.x = r.x+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn composite_unique_does_not_rewrite_when_join_misses_a_key_column() -> Result<()> {+ // The removed side is unique only on the *composite* key `(id, x)`. The+ // join equates `id` but not `x`, so a left row may match many right rows+ // (those sharing its `id` but differing in `x`). Uniqueness on the join+ // keys is not established, so the inner join must be preserved. This+ // guards the requirement that the join cover *every* column of the+ // unique key, not just some.+ let left = scan("l", &test_schema(), Constraints::default())?;+ let right = scan("r", &test_schema(), composite_primary_key_on_id_x())?;+ let plan = LogicalPlanBuilder::from(left)+ .join(right, Inner, (vec!["l.id"], vec!["r.id"]), None)?+ .project(vec![col("l.y")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.y+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn top_n_sort_blocks_duplicate_insensitive_rewrite() -> Result<()> {+ // A top-N `Sort` (one with a `fetch`) makes the row count observable, so+ // the duplicate-insensitivity established by the `GROUP BY` does not survive+ // past it. With a non-unique right side the join must stay an inner join: a+ // semi join could drop matching duplicates and change which rows fall within+ // the top N.+ let plan = left_join_right()?+ .sort_with_limit(vec![col("l.x").sort(true, false)], Some(5))?+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[]]+ Sort: l.x ASC NULLS LAST, fetch=5+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn sort_without_fetch_preserves_duplicate_insensitive_rewrite() -> Result<()> {+ // A `Sort` without a `fetch` does not make the row count observable, so it+ // forwards the parent's duplicate-insensitivity to the join unchanged+ // (sorting before or after duplicate removal is equivalent). The non-unique+ // right side is unreferenced, so the inner join collapses to a semi join.+ let plan = left_join_right()?+ .sort(vec![col("l.x").sort(true, false)])?+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[]]+ Sort: l.x ASC NULLS LAST+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn limit_blocks_duplicate_insensitive_rewrite() -> Result<()> {+ // `LIMIT` makes the row count observable, clearing the duplicate-+ // insensitivity established by the `GROUP BY`. With a non-unique right side+ // the join must stay an inner join, since a semi join could drop matching+ // duplicates and change which rows the limit returns.+ let plan = left_join_right()?+ .limit(0, Some(5))?+ .aggregate(vec![col("l.x")], Vec::<Expr>::new())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Aggregate: groupBy=[[l.x]], aggr=[[]]+ Limit: skip=0, fetch=5+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn repartition_hash_key_keeps_removed_side_live() -> Result<()> {+ // The projection keeps only `l.x`, and the right side is unique (PK), so+ // absent any other use of `r` the inner join would collapse to a semi join.+ // But the `Repartition` hashes on `r.y`, which keeps the right side live, so+ // the join must stay an inner join to preserve `r.y` for the partitioning.+ let plan = left_join_right_with_constraints(primary_key_on_id())?+ .repartition(Partitioning::Hash(vec![col("r.y")], 4))?+ .project(vec![col("l.x")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.x+ Repartition: Hash(r.y) partition_count=4+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn repartition_range_key_keeps_removed_side_live() -> Result<()> {+ // The projection keeps only `l.x`, and the right side is unique (PK), so+ // absent any other use of `r` the inner join would collapse to a semi join.+ // But the `Repartition` ranges on `r.y`, which keeps the right side live, so+ // the join must stay an inner join to preserve `r.y` for the partitioning.+ let plan = left_join_right_with_constraints(primary_key_on_id())?+ .repartition(Partitioning::Range(RangePartitioning::try_new(+ vec![col("r.y").sort(true, true)],+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],+ )?))?+ .project(vec![col("l.x")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.x+ Repartition: Range([r.y ASC NULLS FIRST], [(10)], 2)+ Inner Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn distinct_on_enables_semi_join_rewrite() -> Result<()> {+ // `DISTINCT ON (l.x)` is a no-aggregate `GROUP BY` on the columns it reads,+ // so it makes its input duplicate-insensitive. The non-unique right side is+ // unreferenced, so the inner join collapses to a semi join.+ let plan = left_join_right()?+ .distinct_on(vec![col("l.x")], vec![col("l.x")], None)?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ DistinctOn: on_expr=[[l.x]], select_expr=[[l.x]], sort_expr=[[]]+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ #[test]+ fn existing_semi_join_passes_through_unchanged() -> Result<()> {+ // A join that is already a semi join is threaded through unchanged: the rule+ // only rewrites inner joins. This exercises the context-propagation paths for+ // a non-inner join type, whose existence side contributes no live columns.+ let left = scan("l", &test_schema(), Constraints::default())?;+ let right = scan("r", &test_schema(), Constraints::default())?;+ let plan = LogicalPlanBuilder::from(left)+ .join(+ right,+ JoinType::LeftSemi,+ (vec!["l.id"], vec!["r.id"]),+ None,+ )?+ .project(vec![col("l.x")])?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Projection: l.x+ LeftSemi Join: l.id = r.id+ TableScan: l+ TableScan: r+ ")+ }++ fn left_join_right() -> Result<LogicalPlanBuilder> {+ left_join_right_with_constraints(Constraints::default())+ }++ fn left_join_right_with_constraints(+ right_constraints: Constraints,+ ) -> Result<LogicalPlanBuilder> {+ let left = scan("l", &test_schema(), Constraints::default())?;+ let right = scan("r", &test_schema(), right_constraints)?;++ LogicalPlanBuilder::from(left).join(+ right,+ Inner,+ (vec!["l.id"], vec!["r.id"]),+ None,+ )+ }++ fn left_with_constraints_join_right(+ left_constraints: Constraints,+ ) -> Result<LogicalPlanBuilder> {+ let left = scan("l", &test_schema(), left_constraints)?;+ let right = scan("r", &test_schema(), Constraints::default())?;++ LogicalPlanBuilder::from(left).join(+ right,+ Inner,+ (vec!["l.id"], vec!["r.id"]),+ None,+ )+ }++ fn scan(+ name: &str,+ schema: &Schema,+ constraints: Constraints,+ ) -> Result<datafusion_expr::logical_plan::LogicalPlan> {+ if constraints.is_empty() {+ table_scan(Some(name), schema, None)?.build()+ } else {+ LogicalPlanBuilder::scan(+ name,+ table_source_with_constraints(schema, constraints),+ None,+ )?+ .build()+ }+ }++ fn test_schema() -> Schema {+ Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Int32, true),+ ])+ }++ fn primary_key_on_id() -> Constraints {+ Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])])+ }++ /// A nullable unique key on column `x` (index 1). `Unique` (unlike+ /// `PrimaryKey`) marks the dependency as nullable, which is what gates the+ /// rewrite on the join's `null_equality`.+ fn unique_on_x() -> Constraints {+ Constraints::new_unverified(vec![Constraint::Unique(vec![1])])+ }++ /// A composite primary key spanning columns `id` and `x` (indices 0 and 1).+ fn composite_primary_key_on_id_x() -> Constraints {+ Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0, 1])])+ }}diff --git a/datafusion/optimizer/src/optimize_projections/required_indices.rs b/datafusion/optimizer/src/optimize_projections/required_indices.rsindex 5e73a9fbec..33f0d48721 100644--- a/datafusion/optimizer/src/optimize_projections/required_indices.rs+++ b/datafusion/optimizer/src/optimize_projections/required_indices.rs@@ -17,7 +17,8 @@//! [`RequiredIndices`] helper for OptimizeProjection-use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};+use crate::utils::for_each_referenced_index;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Column, DFSchemaRef, Result};use datafusion_expr::{Expr, LogicalPlan};@@ -112,29 +113,8 @@ impl RequiredIndices {/// * `input_schema`: The input schema to analyze for index requirements./// * `expr`: An expression for which we want to find necessary field indices.fn add_expr(&mut self, input_schema: &DFSchemaRef, expr: &Expr) {- // `apply` does not descend into subqueries, so recurse manually to- // handle those cases.- expr.apply(|e| {- match e {- Expr::Column(c) | Expr::OuterReferenceColumn(_, c) => {- if let Some(idx) = input_schema.maybe_index_of_column(c) {- self.indices.push(idx);- }- }- Expr::ScalarSubquery(sub) => {- self.add_exprs(input_schema, &sub.outer_ref_columns);- }- Expr::Exists(ex) => {- self.add_exprs(input_schema, &ex.subquery.outer_ref_columns);- }- Expr::InSubquery(isq) => {- self.add_exprs(input_schema, &isq.subquery.outer_ref_columns);- }- _ => {}- }- Ok(TreeNodeRecursion::Continue)- })- .expect("traversal is infallible");+ for_each_referenced_index(expr, input_schema, |idx| self.indices.push(idx))+ .expect("traversal is infallible");}/// Like [`Self::add_expr`], but for multiple expressions.diff --git a/datafusion/optimizer/src/utils.rs b/datafusion/optimizer/src/utils.rsindex ad151d1ddb..b29649e9ea 100644--- a/datafusion/optimizer/src/utils.rs+++ b/datafusion/optimizer/src/utils.rs@@ -24,9 +24,10 @@ use arrow::array::{Array, RecordBatch, new_null_array};use arrow::datatypes::{DataType, Field, Schema};use datafusion_common::TableReference;use datafusion_common::cast::as_boolean_array;-use datafusion_common::tree_node::{TransformedResult, TreeNode};+use datafusion_common::tree_node::{TransformedResult, TreeNode, TreeNodeRecursion};use datafusion_common::{Column, DFSchema, Result, ScalarValue};use datafusion_expr::execution_props::ExecutionProps;+use datafusion_expr::expr::{Exists, InSubquery, SetComparison};use datafusion_expr::expr_rewriter::replace_col;use datafusion_expr::{ColumnarValue, Expr, logical_plan::LogicalPlan};use datafusion_physical_expr::create_physical_expr;@@ -37,6 +38,56 @@ use std::sync::Arc;/// as it was initially placed here and then moved elsewhere.pub use datafusion_expr::expr_rewriter::NamePreserver;+/// Invokes `f` with the index, within `schema`, of every column referenced by+/// `expr` — including columns reached through a correlated subquery's outer+/// references. Columns absent from `schema` are skipped.+///+/// A subquery's own plan is intentionally not traversed: its internal columns+/// index into its own schema, not `schema`; only the outer (correlated) columns+/// it references from `schema` are relevant. The comparison expression of an+/// `IN`/set-comparison subquery is reached by the normal expression walk.+///+/// This is the shared primitive behind the top-down "which of a node's output+/// columns does an ancestor still need" analyses, namely+/// [`OptimizeProjections`](crate::optimize_projections::OptimizeProjections)+/// and [`EliminateJoin`](crate::eliminate_join::EliminateJoin). The two keep+/// their own required-index containers (an ordered set vs. a hash set), so this+/// reports indices through a callback rather than populating a shared type.+pub(crate) fn for_each_referenced_index(+ expr: &Expr,+ schema: &DFSchema,+ mut f: impl FnMut(usize),+) -> Result<()> {+ visit_referenced_indices(expr, schema, &mut f)+}++fn visit_referenced_indices(+ expr: &Expr,+ schema: &DFSchema,+ f: &mut dyn FnMut(usize),+) -> Result<()> {+ expr.apply(|expr| {+ match expr {+ Expr::Column(column) | Expr::OuterReferenceColumn(_, column) => {+ if let Some(idx) = schema.maybe_index_of_column(column) {+ f(idx);+ }+ }+ Expr::Exists(Exists { subquery, .. })+ | Expr::InSubquery(InSubquery { subquery, .. })+ | Expr::SetComparison(SetComparison { subquery, .. })+ | Expr::ScalarSubquery(subquery) => {+ for outer in &subquery.outer_ref_columns {+ visit_referenced_indices(outer, schema, f)?;+ }+ }+ _ => {}+ }+ Ok(TreeNodeRecursion::Continue)+ })?;+ Ok(())+}+/// Returns true if `expr` contains all columns in `schema_cols`pub(crate) fn has_all_column_refs(expr: &Expr,diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex b037aef3c2..9be1d39d63 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -1333,19 +1333,57 @@ inner join join_t2 on join_t1.t1_id = join_t2.t2_id----logical_plan01)Aggregate: groupBy=[[join_t1.t1_id]], aggr=[[]]-02)--Projection: join_t1.t1_id-03)----Inner Join: join_t1.t1_id = join_t2.t2_id-04)------TableScan: join_t1 projection=[t1_id]-05)------TableScan: join_t2 projection=[t2_id]+02)--LeftSemi Join: join_t1.t1_id = join_t2.t2_id+03)----TableScan: join_t1 projection=[t1_id]+04)----TableScan: join_t2 projection=[t2_id]physical_plan01)AggregateExec: mode=FinalPartitioned, gby=[t1_id@0 as t1_id], aggr=[]02)--RepartitionExec: partitioning=Hash([t1_id@0], 2), input_partitions=203)----AggregateExec: mode=Partial, gby=[t1_id@0 as t1_id], aggr=[]-04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1_id@0, t2_id@0)], projection=[t1_id@0]+04)------HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(t1_id@0, t2_id@0)]05)--------DataSourceExec: partitions=1, partition_sizes=[1]06)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=107)----------DataSourceExec: partitions=1, partition_sizes=[1]+statement ok+set datafusion.explain.logical_plan_only = true;++# A single `count(DISTINCT col)` over a join whose other side is used only as an+# existence filter can be rewritten to a semi join.+query TT+EXPLAIN+select join_t1.t1_id, count(distinct join_t1.t1_int)+from join_t1+inner join join_t2 on join_t1.t1_id = join_t2.t2_id+group by join_t1.t1_id+----+logical_plan+01)Projection: join_t1.t1_id, count(alias1) AS count(DISTINCT join_t1.t1_int)+02)--Aggregate: groupBy=[[join_t1.t1_id]], aggr=[[count(alias1)]]+03)----Aggregate: groupBy=[[join_t1.t1_id, join_t1.t1_int AS alias1]], aggr=[[]]+04)------LeftSemi Join: join_t1.t1_id = join_t2.t2_id+05)--------TableScan: join_t1 projection=[t1_id, t1_int]+06)--------TableScan: join_t2 projection=[t2_id]++# A similar query with two DISTINCT aggregates is currently not rewritten+# TODO: https://github.com/apache/datafusion/issues/22644+query TT+EXPLAIN+select join_t1.t1_id, count(distinct join_t1.t1_int), count(distinct join_t1.t1_name)+from join_t1+inner join join_t2 on join_t1.t1_id = join_t2.t2_id+group by join_t1.t1_id+----+logical_plan+01)Aggregate: groupBy=[[join_t1.t1_id]], aggr=[[count(DISTINCT join_t1.t1_int), count(DISTINCT join_t1.t1_name)]]+02)--Projection: join_t1.t1_id, join_t1.t1_name, join_t1.t1_int+03)----Inner Join: join_t1.t1_id = join_t2.t2_id+04)------TableScan: join_t1 projection=[t1_id, t1_name, t1_int]+05)------TableScan: join_t2 projection=[t2_id]++statement ok+set datafusion.explain.logical_plan_only = false;+# Join on structquery TTexplain select join_t3.s3, join_t4.s4@@ -1411,10 +1449,9 @@ logical_plan01)Projection: count(alias1) AS count(DISTINCT join_t1.t1_id)02)--Aggregate: groupBy=[[]], aggr=[[count(alias1)]]03)----Aggregate: groupBy=[[join_t1.t1_id AS alias1]], aggr=[[]]-04)------Projection: join_t1.t1_id-05)--------Inner Join: join_t1.t1_id = join_t2.t2_id-06)----------TableScan: join_t1 projection=[t1_id]-07)----------TableScan: join_t2 projection=[t2_id]+04)------LeftSemi Join: join_t1.t1_id = join_t2.t2_id+05)--------TableScan: join_t1 projection=[t1_id]+06)--------TableScan: join_t2 projection=[t2_id]physical_plan01)ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT join_t1.t1_id)]02)--AggregateExec: mode=Final, gby=[], aggr=[count(alias1)]@@ -1423,7 +1460,7 @@ physical_plan05)--------AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[]06)----------RepartitionExec: partitioning=Hash([alias1@0], 2), input_partitions=207)------------AggregateExec: mode=Partial, gby=[t1_id@0 as alias1], aggr=[]-08)--------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1_id@0, t2_id@0)], projection=[t1_id@0]+08)--------------HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(t1_id@0, t2_id@0)]09)----------------DataSourceExec: partitions=1, partition_sizes=[1]10)----------------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=111)------------------DataSourceExec: partitions=1, partition_sizes=[1]diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 3d6f802745..d305109a48 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -338,13 +338,13 @@ where c_acctbal < (logical_plan01)Sort: customer.c_custkey ASC NULLS LAST02)--Projection: customer.c_custkey-03)----Inner Join: customer.c_custkey = __scalar_sq_1.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_1.sum(orders.o_totalprice)+03)----LeftSemi Join: customer.c_custkey = __scalar_sq_1.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_1.sum(orders.o_totalprice)04)------TableScan: customer projection=[c_custkey, c_acctbal]05)------SubqueryAlias: __scalar_sq_106)--------Projection: sum(orders.o_totalprice), orders.o_custkey07)----------Aggregate: groupBy=[[orders.o_custkey]], aggr=[[sum(orders.o_totalprice)]]08)------------Projection: orders.o_custkey, orders.o_totalprice-09)--------------Inner Join: orders.o_orderkey = __scalar_sq_2.l_orderkey Filter: CAST(orders.o_totalprice AS Decimal128(25, 2)) < __scalar_sq_2.price+09)--------------LeftSemi Join: orders.o_orderkey = __scalar_sq_2.l_orderkey Filter: CAST(orders.o_totalprice AS Decimal128(25, 2)) < __scalar_sq_2.price10)----------------TableScan: orders projection=[o_orderkey, o_custkey, o_totalprice]11)----------------SubqueryAlias: __scalar_sq_212)------------------Projection: sum(lineitem.l_extendedprice) AS price, lineitem.l_orderkey@@ -555,7 +555,7 @@ logical_plan02)--TableScan: t0 projection=[t0_id, t0_name]03)--SubqueryAlias: __correlated_sq_204)----Projection: t1.t1_name-05)------Inner Join: t1.t1_id = t2.t2_id+05)------LeftSemi Join: t1.t1_id = t2.t2_id06)--------TableScan: t1 projection=[t1_id, t1_name]07)--------TableScan: t2 projection=[t2_id]@@ -568,7 +568,7 @@ logical_plan02)--TableScan: t0 projection=[t0_id, t0_name]03)--SubqueryAlias: __correlated_sq_104)----Projection: t2.t2_name-05)------Inner Join: t1.t1_id = t2.t2_id+05)------RightSemi Join: t1.t1_id = t2.t2_id06)--------TableScan: t1 projection=[t1_id]07)--------SubqueryAlias: t208)----------TableScan: t2 projection=[t2_id, t2_name]@@ -1675,7 +1675,7 @@ where c_acctbal < (logical_plan01)Sort: customer.c_custkey ASC NULLS LAST02)--Projection: customer.c_custkey-03)----Inner Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)+03)----LeftSemi Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)04)------TableScan: customer projection=[c_custkey, c_acctbal]05)------SubqueryAlias: __scalar_sq_206)--------Projection: sum(orders.o_totalprice), orders.o_custkey@@ -1701,7 +1701,7 @@ where c_acctbal < (logical_plan01)Sort: customer.c_custkey ASC NULLS LAST02)--Projection: customer.c_custkey-03)----Inner Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)+03)----LeftSemi Join: customer.c_custkey = __scalar_sq_2.o_custkey Filter: CAST(customer.c_acctbal AS Decimal128(25, 2)) < __scalar_sq_2.sum(orders.o_totalprice)04)------TableScan: customer projection=[c_custkey, c_acctbal]05)------SubqueryAlias: __scalar_sq_206)--------Projection: sum(orders.o_totalprice), orders.o_custkey@@ -1746,7 +1746,7 @@ WHERE e1.salary > (----logical_plan01)Projection: e1.employee_name, e1.salary-02)--Inner Join: e1.dept_id = __scalar_sq_1.dept_id Filter: CAST(e1.salary AS Decimal128(38, 14)) > __scalar_sq_1.avg(e2.salary)+02)--LeftSemi Join: e1.dept_id = __scalar_sq_1.dept_id Filter: CAST(e1.salary AS Decimal128(38, 14)) > __scalar_sq_1.avg(e2.salary)03)----SubqueryAlias: e104)------TableScan: employees projection=[employee_name, dept_id, salary]05)----SubqueryAlias: __scalar_sq_1diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.partindex 6bab765c67..cd86b618f0 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part@@ -54,7 +54,7 @@ logical_plan05)--------Projection: CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty) AS Float64) * Float64(0.0001) AS Decimal128(38, 15))06)----------Aggregate: groupBy=[[]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]07)------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost-08)--------------Inner Join: supplier.s_nationkey = nation.n_nationkey+08)--------------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey09)----------------Projection: partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey10)------------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey11)--------------------TableScan: partsupp projection=[ps_suppkey, ps_availqty, ps_supplycost]@@ -64,7 +64,7 @@ logical_plan15)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY")]16)------Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[sum(partsupp.ps_supplycost * CAST(partsupp.ps_availqty AS Decimal128(10, 0)))]]17)--------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost-18)----------Inner Join: supplier.s_nationkey = nation.n_nationkey+18)----------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey19)------------Projection: partsupp.ps_partkey, partsupp.ps_availqty, partsupp.ps_supplycost, supplier.s_nationkey20)--------------Inner Join: partsupp.ps_suppkey = supplier.s_suppkey21)----------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost]@@ -81,7 +81,7 @@ physical_plan06)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]07)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=408)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-09)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]+09)----------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]10)------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=411)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]12)----------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4@@ -96,7 +96,7 @@ physical_plan21)----AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]22)------CoalescePartitionsExec23)--------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-24)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]+24)----------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]25)------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=426)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]27)----------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.partindex 9526d85319..7ef36a72ec 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part@@ -57,7 +57,7 @@ logical_plan01)Projection: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS revenue02)--Aggregate: groupBy=[[]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(1,20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]03)----Projection: lineitem.l_extendedprice, lineitem.l_discount-04)------Inner Join: lineitem.l_partkey = part.p_partkey Filter: part.p_brand = Utf8View("Brand#12") AND part.p_container IN ([Utf8View("SM CASE"), Utf8View("SM BOX"), Utf8View("SM PACK"), Utf8View("SM PKG")]) AND lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_container IN ([Utf8View("MED BAG"), Utf8View("MED BOX"), Utf8View("MED PKG"), Utf8View("MED PACK")]) AND lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_container IN ([Utf8View("LG CASE"), Utf8View("LG BOX"), Utf8View("LG PACK"), Utf8View("LG PKG")]) AND lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2) AND part.p_size <= Int32(15)+04)------LeftSemi Join: lineitem.l_partkey = part.p_partkey Filter: part.p_brand = Utf8View("Brand#12") AND part.p_container IN ([Utf8View("SM CASE"), Utf8View("SM BOX"), Utf8View("SM PACK"), Utf8View("SM PKG")]) AND lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_container IN ([Utf8View("MED BAG"), Utf8View("MED BOX"), Utf8View("MED PKG"), Utf8View("MED PACK")]) AND lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_container IN ([Utf8View("LG CASE"), Utf8View("LG BOX"), Utf8View("LG PACK"), Utf8View("LG PKG")]) AND lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2) AND part.p_size <= Int32(15)05)--------Projection: lineitem.l_partkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount06)----------Filter: (lineitem.l_shipmode = Utf8View("AIR") OR lineitem.l_shipmode = Utf8View("AIR REG")) AND lineitem.l_shipinstruct = Utf8View("DELIVER IN PERSON") AND (lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) OR lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) OR lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2))07)------------TableScan: lineitem projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View("AIR") OR lineitem.l_shipmode = Utf8View("AIR REG"), lineitem.l_shipinstruct = Utf8View("DELIVER IN PERSON"), lineitem.l_quantity >= Decimal128(1.00,15,2) AND lineitem.l_quantity <= Decimal128(11.00,15,2) OR lineitem.l_quantity >= Decimal128(10.00,15,2) AND lineitem.l_quantity <= Decimal128(20.00,15,2) OR lineitem.l_quantity >= Decimal128(20.00,15,2) AND lineitem.l_quantity <= Decimal128(30.00,15,2)]@@ -68,7 +68,7 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= 1.00 AND l_quantity@0 <= 11.00 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= 10.00 AND l_quantity@0 <= 20.00 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= 20.00 AND l_quantity@0 <= 30.00 AND p_size@2 <= 15, projection=[l_extendedprice@2, l_discount@3]+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= 1.00 AND l_quantity@0 <= 11.00 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= 10.00 AND l_quantity@0 <= 20.00 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= 20.00 AND l_quantity@0 <= 30.00 AND p_size@2 <= 15, projection=[l_extendedprice@2, l_discount@3]06)----------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=407)------------FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= 1.00 AND l_quantity@1 <= 11.00 OR l_quantity@1 >= 10.00 AND l_quantity@1 <= 20.00 OR l_quantity@1 >= 20.00 AND l_quantity@1 <= 30.00), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3]08)--------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], constraints=[PrimaryKey([0, 3])], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.partindex 31702ab39e..b6fa1c4806 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part@@ -65,9 +65,9 @@ limit 10;logical_plan01)Sort: supplier.s_acctbal DESC NULLS FIRST, nation.n_name ASC NULLS LAST, supplier.s_name ASC NULLS LAST, part.p_partkey ASC NULLS LAST, fetch=1002)--Projection: supplier.s_acctbal, supplier.s_name, nation.n_name, part.p_partkey, part.p_mfgr, supplier.s_address, supplier.s_phone, supplier.s_comment-03)----Inner Join: part.p_partkey = __scalar_sq_1.ps_partkey, partsupp.ps_supplycost = __scalar_sq_1.min(partsupp.ps_supplycost)+03)----LeftSemi Join: part.p_partkey = __scalar_sq_1.ps_partkey, partsupp.ps_supplycost = __scalar_sq_1.min(partsupp.ps_supplycost)04)------Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_supplycost, nation.n_name-05)--------Inner Join: nation.n_regionkey = region.r_regionkey+05)--------LeftSemi Join: nation.n_regionkey = region.r_regionkey06)----------Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_supplycost, nation.n_name, nation.n_regionkey07)------------Inner Join: supplier.s_nationkey = nation.n_nationkey08)--------------Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_nationkey, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_supplycost@@ -87,7 +87,7 @@ logical_plan22)--------Projection: min(partsupp.ps_supplycost), partsupp.ps_partkey23)----------Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[min(partsupp.ps_supplycost)]]24)------------Projection: partsupp.ps_partkey, partsupp.ps_supplycost-25)--------------Inner Join: nation.n_regionkey = region.r_regionkey+25)--------------LeftSemi Join: nation.n_regionkey = region.r_regionkey26)----------------Projection: partsupp.ps_partkey, partsupp.ps_supplycost, nation.n_regionkey27)------------------Inner Join: supplier.s_nationkey = nation.n_nationkey28)--------------------Projection: partsupp.ps_partkey, partsupp.ps_supplycost, supplier.s_nationkey@@ -101,9 +101,9 @@ logical_planphysical_plan01)SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], fetch=1002)--SortExec: TopK(fetch=10), expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true]-03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[s_acctbal@5, s_name@2, n_name@8, p_partkey@0, p_mfgr@1, s_address@3, s_phone@4, s_comment@6]+03)----HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[s_acctbal@5, s_name@2, n_name@8, p_partkey@0, p_mfgr@1, s_address@3, s_phone@4, s_comment@6]04)------RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 4), input_partitions=4-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@9, r_regionkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, ps_supplycost@7, n_name@8]+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@9, r_regionkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, ps_supplycost@7, n_name@8]06)----------RepartitionExec: partitioning=Hash([n_regionkey@9], 4), input_partitions=407)------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@4, n_nationkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@10, n_regionkey@11]08)--------------RepartitionExec: partitioning=Hash([s_nationkey@4], 4), input_partitions=4@@ -128,7 +128,7 @@ physical_plan27)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]28)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=429)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]-30)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@2, r_regionkey@0)], projection=[ps_partkey@0, ps_supplycost@1]+30)----------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@2, r_regionkey@0)], projection=[ps_partkey@0, ps_supplycost@1]31)------------------RepartitionExec: partitioning=Hash([n_regionkey@2], 4), input_partitions=432)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_partkey@0, ps_supplycost@1, n_regionkey@4]33)----------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.partindex ad65a4f08a..e038a7482d 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q20.slt.part@@ -60,14 +60,14 @@ logical_plan02)--Projection: supplier.s_name, supplier.s_address03)----LeftSemi Join: supplier.s_suppkey = __correlated_sq_2.ps_suppkey04)------Projection: supplier.s_suppkey, supplier.s_name, supplier.s_address-05)--------Inner Join: supplier.s_nationkey = nation.n_nationkey+05)--------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey06)----------TableScan: supplier projection=[s_suppkey, s_name, s_address, s_nationkey]07)----------Projection: nation.n_nationkey08)------------Filter: nation.n_name = Utf8View("CANADA")09)--------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("CANADA")]10)------SubqueryAlias: __correlated_sq_211)--------Projection: partsupp.ps_suppkey-12)----------Inner Join: partsupp.ps_partkey = __scalar_sq_3.l_partkey, partsupp.ps_suppkey = __scalar_sq_3.l_suppkey Filter: CAST(partsupp.ps_availqty AS Float64) > __scalar_sq_3.Float64(0.5) * sum(lineitem.l_quantity)+12)----------LeftSemi Join: partsupp.ps_partkey = __scalar_sq_3.l_partkey, partsupp.ps_suppkey = __scalar_sq_3.l_suppkey Filter: CAST(partsupp.ps_availqty AS Float64) > __scalar_sq_3.Float64(0.5) * sum(lineitem.l_quantity)13)------------LeftSemi Join: partsupp.ps_partkey = __correlated_sq_1.p_partkey14)--------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_availqty]15)--------------SubqueryAlias: __correlated_sq_1@@ -85,7 +85,7 @@ physical_plan02)--SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true]03)----HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_name@1, s_address@2]04)------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=4-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[s_suppkey@0, s_name@1, s_address@2]+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@3, n_nationkey@0)], projection=[s_suppkey@0, s_name@1, s_address@2]06)----------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=107)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], constraints=[PrimaryKey([0])], file_type=csv, has_header=false08)----------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4@@ -93,7 +93,7 @@ physical_plan10)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=111)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], constraints=[PrimaryKey([0])], file_type=csv, has_header=false12)------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4-13)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1]+13)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1]14)----------RepartitionExec: partitioning=Hash([ps_partkey@0, ps_suppkey@1], 4), input_partitions=415)------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ps_partkey@0, p_partkey@0)]16)--------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.partindex 2001aa8df0..812f5d2cba 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part@@ -65,30 +65,29 @@ logical_plan05)--------LeftAnti Join: l1.l_orderkey = __correlated_sq_2.l_orderkey Filter: __correlated_sq_2.l_suppkey != l1.l_suppkey06)----------LeftSemi Join: l1.l_orderkey = __correlated_sq_1.l_orderkey Filter: __correlated_sq_1.l_suppkey != l1.l_suppkey07)------------Projection: supplier.s_name, l1.l_orderkey, l1.l_suppkey-08)--------------Inner Join: supplier.s_nationkey = nation.n_nationkey-09)----------------Projection: supplier.s_name, supplier.s_nationkey, l1.l_orderkey, l1.l_suppkey-10)------------------Inner Join: l1.l_orderkey = orders.o_orderkey-11)--------------------Projection: supplier.s_name, supplier.s_nationkey, l1.l_orderkey, l1.l_suppkey-12)----------------------Inner Join: supplier.s_suppkey = l1.l_suppkey-13)------------------------TableScan: supplier projection=[s_suppkey, s_name, s_nationkey]-14)------------------------SubqueryAlias: l1-15)--------------------------Projection: lineitem.l_orderkey, lineitem.l_suppkey-16)----------------------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate-17)------------------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]-18)--------------------Projection: orders.o_orderkey-19)----------------------Filter: orders.o_orderstatus = Utf8View("F")-20)------------------------TableScan: orders projection=[o_orderkey, o_orderstatus], partial_filters=[orders.o_orderstatus = Utf8View("F")]-21)----------------Projection: nation.n_nationkey-22)------------------Filter: nation.n_name = Utf8View("SAUDI ARABIA")-23)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("SAUDI ARABIA")]-24)------------SubqueryAlias: __correlated_sq_1-25)--------------SubqueryAlias: l2-26)----------------TableScan: lineitem projection=[l_orderkey, l_suppkey]-27)----------SubqueryAlias: __correlated_sq_2-28)------------SubqueryAlias: l3-29)--------------Projection: lineitem.l_orderkey, lineitem.l_suppkey-30)----------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate-31)------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]+08)--------------LeftSemi Join: supplier.s_nationkey = nation.n_nationkey+09)----------------LeftSemi Join: l1.l_orderkey = orders.o_orderkey+10)------------------Projection: supplier.s_name, supplier.s_nationkey, l1.l_orderkey, l1.l_suppkey+11)--------------------Inner Join: supplier.s_suppkey = l1.l_suppkey+12)----------------------TableScan: supplier projection=[s_suppkey, s_name, s_nationkey]+13)----------------------SubqueryAlias: l1+14)------------------------Projection: lineitem.l_orderkey, lineitem.l_suppkey+15)--------------------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate+16)----------------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]+17)------------------Projection: orders.o_orderkey+18)--------------------Filter: orders.o_orderstatus = Utf8View("F")+19)----------------------TableScan: orders projection=[o_orderkey, o_orderstatus], partial_filters=[orders.o_orderstatus = Utf8View("F")]+20)----------------Projection: nation.n_nationkey+21)------------------Filter: nation.n_name = Utf8View("SAUDI ARABIA")+22)--------------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("SAUDI ARABIA")]+23)------------SubqueryAlias: __correlated_sq_1+24)--------------SubqueryAlias: l2+25)----------------TableScan: lineitem projection=[l_orderkey, l_suppkey]+26)----------SubqueryAlias: __correlated_sq_2+27)------------SubqueryAlias: l3+28)--------------Projection: lineitem.l_orderkey, lineitem.l_suppkey+29)----------------Filter: lineitem.l_receiptdate > lineitem.l_commitdate+30)------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]physical_plan01)SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST]02)--SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true]@@ -99,9 +98,9 @@ physical_plan07)------------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0]08)--------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@009)----------------RepartitionExec: partitioning=Hash([l_orderkey@1], 4), input_partitions=4-10)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@1, n_nationkey@0)], projection=[s_name@0, l_orderkey@2, l_suppkey@3]+10)------------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_nationkey@1, n_nationkey@0)], projection=[s_name@0, l_orderkey@2, l_suppkey@3]11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@1], 4), input_partitions=4-12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@2, o_orderkey@0)], projection=[s_name@0, s_nationkey@1, l_orderkey@2, l_suppkey@3]+12)----------------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(l_orderkey@2, o_orderkey@0)]13)------------------------RepartitionExec: partitioning=Hash([l_orderkey@2], 4), input_partitions=414)--------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4]15)----------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.partindex 7a3523b088..a92a752211 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part@@ -48,7 +48,7 @@ logical_plan04)------Projection: orders.o_orderdate, orders.o_shippriority, lineitem.l_orderkey, lineitem.l_extendedprice, lineitem.l_discount05)--------Inner Join: orders.o_orderkey = lineitem.l_orderkey06)----------Projection: orders.o_orderkey, orders.o_orderdate, orders.o_shippriority-07)------------Inner Join: customer.c_custkey = orders.o_custkey+07)------------RightSemi Join: customer.c_custkey = orders.o_custkey08)--------------Projection: customer.c_custkey09)----------------Filter: customer.c_mktsegment = Utf8View("BUILDING")10)------------------TableScan: customer projection=[c_custkey, c_mktsegment], partial_filters=[customer.c_mktsegment = Utf8View("BUILDING")]@@ -64,7 +64,7 @@ physical_plan04)------AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5]06)----------RepartitionExec: partitioning=Hash([o_orderkey@0], 4), input_partitions=4-07)------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4]+07)------------HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@0, o_orderdate@2, o_shippriority@3]08)--------------RepartitionExec: partitioning=Hash([c_custkey@0], 4), input_partitions=409)----------------FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0]10)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:0..606529], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:606529..1213058], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:1213058..1819587], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl:1819587..2426114]]}, projection=[c_custkey, c_mktsegment], constraints=[PrimaryKey([0])], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.partindex 6dd06b269e..036c0e3b8c 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part@@ -47,7 +47,7 @@ logical_plan02)--Projection: nation.n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount) AS revenue03)----Aggregate: groupBy=[[nation.n_name]], aggr=[[sum(lineitem.l_extendedprice * (Decimal128(1,20,0) - lineitem.l_discount)) AS sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]]04)------Projection: lineitem.l_extendedprice, lineitem.l_discount, nation.n_name-05)--------Inner Join: nation.n_regionkey = region.r_regionkey+05)--------LeftSemi Join: nation.n_regionkey = region.r_regionkey06)----------Projection: lineitem.l_extendedprice, lineitem.l_discount, nation.n_name, nation.n_regionkey07)------------Inner Join: supplier.s_nationkey = nation.n_nationkey08)--------------Projection: lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey@@ -73,7 +73,7 @@ physical_plan04)------AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]05)--------RepartitionExec: partitioning=Hash([n_name@0], 4), input_partitions=406)----------AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-07)------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)], projection=[l_extendedprice@0, l_discount@1, n_name@2]+07)------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@3, r_regionkey@0)], projection=[l_extendedprice@0, l_discount@1, n_name@2]08)--------------RepartitionExec: partitioning=Hash([n_regionkey@3], 4), input_partitions=409)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[l_extendedprice@0, l_discount@1, n_name@4, n_regionkey@5]10)------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.partindex c38930cb5b..902413e9ef 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part@@ -61,7 +61,7 @@ logical_plan03)----Aggregate: groupBy=[[all_nations.o_year]], aggr=[[sum(CASE WHEN all_nations.nation = Utf8View("BRAZIL") THEN all_nations.volume ELSE Decimal128(0.0000,38,4) END) AS sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]]04)------SubqueryAlias: all_nations05)--------Projection: date_part(Utf8("YEAR"), orders.o_orderdate) AS o_year, lineitem.l_extendedprice * (Decimal128(1,20,0) - lineitem.l_discount) AS volume, n2.n_name AS nation-06)----------Inner Join: n1.n_regionkey = region.r_regionkey+06)----------LeftSemi Join: n1.n_regionkey = region.r_regionkey07)------------Projection: lineitem.l_extendedprice, lineitem.l_discount, orders.o_orderdate, n1.n_regionkey, n2.n_name08)--------------Inner Join: supplier.s_nationkey = n2.n_nationkey09)----------------Projection: lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey, orders.o_orderdate, n1.n_regionkey@@ -73,7 +73,7 @@ logical_plan15)----------------------------Projection: lineitem.l_orderkey, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey16)------------------------------Inner Join: lineitem.l_suppkey = supplier.s_suppkey17)--------------------------------Projection: lineitem.l_orderkey, lineitem.l_suppkey, lineitem.l_extendedprice, lineitem.l_discount-18)----------------------------------Inner Join: part.p_partkey = lineitem.l_partkey+18)----------------------------------RightSemi Join: part.p_partkey = lineitem.l_partkey19)------------------------------------Projection: part.p_partkey20)--------------------------------------Filter: part.p_type = Utf8View("ECONOMY ANODIZED STEEL")21)----------------------------------------TableScan: part projection=[p_partkey, p_type], partial_filters=[part.p_type = Utf8View("ECONOMY ANODIZED STEEL")]@@ -97,7 +97,7 @@ physical_plan05)--------RepartitionExec: partitioning=Hash([o_year@0], 4), input_partitions=406)----------AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE 0.0000 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]07)------------ProjectionExec: expr=[date_part(YEAR, o_orderdate@0) as o_year, l_extendedprice@1 * (1 - l_discount@2) as volume, n_name@3 as nation]-08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)], projection=[o_orderdate@2, l_extendedprice@0, l_discount@1, n_name@4]+08)--------------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(n_regionkey@3, r_regionkey@0)], projection=[o_orderdate@2, l_extendedprice@0, l_discount@1, n_name@4]09)----------------RepartitionExec: partitioning=Hash([n_regionkey@3], 4), input_partitions=410)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[l_extendedprice@0, l_discount@1, o_orderdate@3, n_regionkey@4, n_name@6]11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4@@ -109,7 +109,7 @@ physical_plan17)--------------------------------RepartitionExec: partitioning=Hash([l_orderkey@0], 4), input_partitions=418)----------------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@1, s_suppkey@0)], projection=[l_orderkey@0, l_extendedprice@2, l_discount@3, s_nationkey@5]19)------------------------------------RepartitionExec: partitioning=Hash([l_suppkey@1], 4), input_partitions=4-20)--------------------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5]+20)--------------------------------------HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@0, l_suppkey@2, l_extendedprice@3, l_discount@4]21)----------------------------------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=422)------------------------------------------FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0]23)--------------------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:0..597773], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:597773..1195546], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1195546..1793319], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1793319..2391090]]}, projection=[p_partkey, p_type], constraints=[PrimaryKey([0])], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.partindex ca09252a4b..ade7ed7a6c 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part@@ -64,16 +64,15 @@ logical_plan10)------------------Inner Join: lineitem.l_suppkey = partsupp.ps_suppkey, lineitem.l_partkey = partsupp.ps_partkey11)--------------------Projection: lineitem.l_orderkey, lineitem.l_partkey, lineitem.l_suppkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey12)----------------------Inner Join: lineitem.l_suppkey = supplier.s_suppkey-13)------------------------Projection: lineitem.l_orderkey, lineitem.l_partkey, lineitem.l_suppkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount-14)--------------------------Inner Join: part.p_partkey = lineitem.l_partkey-15)----------------------------Projection: part.p_partkey-16)------------------------------Filter: part.p_name LIKE Utf8View("%green%")-17)--------------------------------TableScan: part projection=[p_partkey, p_name], partial_filters=[part.p_name LIKE Utf8View("%green%")]-18)----------------------------TableScan: lineitem projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount]-19)------------------------TableScan: supplier projection=[s_suppkey, s_nationkey]-20)--------------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]-21)----------------TableScan: orders projection=[o_orderkey, o_orderdate]-22)------------TableScan: nation projection=[n_nationkey, n_name]+13)------------------------RightSemi Join: part.p_partkey = lineitem.l_partkey+14)--------------------------Projection: part.p_partkey+15)----------------------------Filter: part.p_name LIKE Utf8View("%green%")+16)------------------------------TableScan: part projection=[p_partkey, p_name], partial_filters=[part.p_name LIKE Utf8View("%green%")]+17)--------------------------TableScan: lineitem projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount]+18)------------------------TableScan: supplier projection=[s_suppkey, s_nationkey]+19)--------------------TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]+20)----------------TableScan: orders projection=[o_orderkey, o_orderdate]+21)------------TableScan: nation projection=[n_nationkey, n_name]physical_plan01)SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC], fetch=1002)--SortExec: TopK(fetch=10), expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true]@@ -90,7 +89,7 @@ physical_plan13)------------------------RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 4), input_partitions=414)--------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, s_suppkey@0)], projection=[l_orderkey@0, l_partkey@1, l_suppkey@2, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@7]15)----------------------------RepartitionExec: partitioning=Hash([l_suppkey@2], 4), input_partitions=4-16)------------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6]+16)------------------------------HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(p_partkey@0, l_partkey@1)]17)--------------------------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=418)----------------------------------FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0]19)------------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:0..597773], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:597773..1195546], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1195546..1793319], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl:1793319..2391090]]}, projection=[p_partkey, p_name], constraints=[PrimaryKey([0])], file_type=csv, has_header=false