Apache DataFusion SQL Query Engine
fix: handle interleaved HashJoin projections in sort pushdown (#23591)
xudong.wa287c0a95931
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex 8e8d222bb0..e9ad978b2e 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -33,7 +33,7 @@ use arrow::compute::{SortOptions};use arrow::datatypes::{DataType, SchemaRef};use datafusion_common::config::{ConfigOptions, CsvOptions};use datafusion_common::tree_node::{TreeNode, TransformedResult};-use datafusion_common::{create_array, Result, TableReference};+use datafusion_common::{create_array, NullEquality, Result, TableReference};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;use datafusion_expr_common::operator::Operator;@@ -44,6 +44,7 @@ use datafusion_physical_expr_common::sort_expr::{};use datafusion_physical_expr::{Distribution, Partitioning, PhysicalExpr};use datafusion_physical_expr::expressions::{col, BinaryExpr, Column, NotExpr};+use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;@@ -237,6 +238,48 @@ async fn test_remove_unnecessary_sort5() -> Result<()> {Ok(())}+#[tokio::test]+async fn test_hash_join_interleaved_projection_preserves_parent_sort() -> Result<()> {+ let left_schema = create_test_schema()?;+ let right_schema = create_test_schema2()?;+ let left = parquet_exec(left_schema.clone());+ let right = parquet_exec(right_schema.clone());+ let on = vec![(+ Arc::new(Column::new_with_schema("nullable_col", &left_schema)?) as _,+ Arc::new(Column::new_with_schema("col_a", &right_schema)?) as _,+ )];+ let join = Arc::new(HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::Right,+ // Interleave a right-side column before a left-side column.+ Some(vec![2, 0]),+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ false,+ )?);+ let ordering = [sort_expr("nullable_col", &join.schema())].into();+ let physical_plan = sort_exec(ordering, join);++ let mut config = ConfigOptions::new();+ config.execution.target_partitions = 10;+ let optimized_plan =+ EnsureRequirements::new().optimize(Arc::clone(&physical_plan), &config)?;+ let optimized_plan = SanityCheckPlan::new().optimize(optimized_plan, &config)?;++ assert_snapshot!(displayable(optimized_plan.as_ref()).indent(true), @r"+ SortPreservingMergeExec: [nullable_col@1 ASC]+ SortExec: expr=[nullable_col@1 ASC], preserve_partitioning=[true]+ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(nullable_col@0, col_a@0)], projection=[col_a@2, nullable_col@0]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquet+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[col_a, col_b], file_type=parquet+ ");+ Ok(())+}+#[tokio::test]async fn test_do_not_remove_sort_with_limit() -> Result<()> {let schema = create_test_schema()?;diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex c1e42a7c9a..5c17ffbd1e 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -983,12 +983,11 @@ fn handle_hash_join(} else {column_indices.iter().collect()};- let len_of_left_fields = projected_indices- .iter()- .filter(|ci| ci.side == JoinSide::Left)- .count();-- let all_from_right_child = all_indices.iter().all(|i| *i >= len_of_left_fields);+ let all_from_right_child = all_indices.iter().all(|i| {+ projected_indices+ .get(*i)+ .is_some_and(|ci| ci.side == JoinSide::Right)+ });let plan_children = plan.children();