Skip to content

Apache DataFusion SQL Query Engine

fix: handle interleaved HashJoin projections in sort pushdown (#23591)

xudong.wa287c0a95931
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
index 8e8d222bb0..e9ad978b2e 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
@@ -33,7 +33,7 @@ use arrow::compute::{SortOptions};
use arrow::datatypes::{DataType, SchemaRef};
use datafusion_common::config::{ConfigOptions, CsvOptions};
use datafusion_common::tree_node::{TreeNode, TransformedResult};
-use datafusion_common::{create_array, Result, TableReference};
+use datafusion_common::{create_array, NullEquality, Result, TableReference};
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
use datafusion_datasource::source::DataSourceExec;
use datafusion_expr_common::operator::Operator;
@@ -44,6 +44,7 @@ use datafusion_physical_expr_common::sort_expr::{
};
use datafusion_physical_expr::{Distribution, Partitioning, PhysicalExpr};
use datafusion_physical_expr::expressions::{col, BinaryExpr, Column, NotExpr};
+use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
@@ -237,6 +238,48 @@ async fn test_remove_unnecessary_sort5() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn test_hash_join_interleaved_projection_preserves_parent_sort() -> Result<()> {
+ let left_schema = create_test_schema()?;
+ let right_schema = create_test_schema2()?;
+ let left = parquet_exec(left_schema.clone());
+ let right = parquet_exec(right_schema.clone());
+ let on = vec![(
+ Arc::new(Column::new_with_schema("nullable_col", &left_schema)?) as _,
+ Arc::new(Column::new_with_schema("col_a", &right_schema)?) as _,
+ )];
+ let join = Arc::new(HashJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ &JoinType::Right,
+ // Interleave a right-side column before a left-side column.
+ Some(vec![2, 0]),
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ false,
+ )?);
+ let ordering = [sort_expr("nullable_col", &join.schema())].into();
+ let physical_plan = sort_exec(ordering, join);
+
+ let mut config = ConfigOptions::new();
+ config.execution.target_partitions = 10;
+ let optimized_plan =
+ EnsureRequirements::new().optimize(Arc::clone(&physical_plan), &config)?;
+ let optimized_plan = SanityCheckPlan::new().optimize(optimized_plan, &config)?;
+
+ assert_snapshot!(displayable(optimized_plan.as_ref()).indent(true), @r"
+ SortPreservingMergeExec: [nullable_col@1 ASC]
+ SortExec: expr=[nullable_col@1 ASC], preserve_partitioning=[true]
+ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(nullable_col@0, col_a@0)], projection=[col_a@2, nullable_col@0]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquet
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[col_a, col_b], file_type=parquet
+ ");
+ Ok(())
+}
+
#[tokio::test]
async fn test_do_not_remove_sort_with_limit() -> Result<()> {
let schema = create_test_schema()?;
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index c1e42a7c9a..5c17ffbd1e 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -983,12 +983,11 @@ fn handle_hash_join(
} else {
column_indices.iter().collect()
};
- let len_of_left_fields = projected_indices
- .iter()
- .filter(|ci| ci.side == JoinSide::Left)
- .count();
-
- let all_from_right_child = all_indices.iter().all(|i| *i >= len_of_left_fields);
+ let all_from_right_child = all_indices.iter().all(|i| {
+ projected_indices
+ .get(*i)
+ .is_some_and(|ci| ci.side == JoinSide::Right)
+ });
let plan_children = plan.children();