Apache DataFusion SQL Query Engine
fix: reject unbounded null-aware joins (#27256)
Phoenixf94aaa187658
diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rsindex b8a3aefbe8..2f4eef0cbf 100644--- a/datafusion/physical-optimizer/src/join_selection.rs+++ b/datafusion/physical-optimizer/src/join_selection.rs@@ -16,7 +16,7 @@ use datafusion_common::Statistics;use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::{JoinSide, JoinType, internal_err};+use datafusion_common::{JoinSide, JoinType, internal_err, plan_err};use datafusion_expr_common::sort_properties::SortProperties;use datafusion_physical_expr::LexOrdering;use datafusion_physical_expr::expressions::Column;@@ -367,6 +367,11 @@ pub type PipelineFixerSubrule =/// An `Option` that contains the `Result` of the transformation. If the transformation is not applicable,/// it returns `None`. If applicable, it returns `Some(Ok(...))` with the modified pipeline state,/// or `Some(Err(...))` if an error occurs during the transformation.+///+/// # Errors+/// Returns a planning error if both inputs are unbounded and emit incrementally,+/// and the join is null-aware. The symmetric hash join used for these inputs+/// does not support null-aware semantics.fn hash_join_convert_symmetric_subrule(input: Arc<dyn ExecutionPlan>,config_options: &ConfigOptions,@@ -385,6 +390,12 @@ fn hash_join_convert_symmetric_subrule();// Process only if both left and right sides are unbounded and incrementally emit.if left_unbounded && right_unbounded & left_incremental & right_incremental {+ if hash_join.null_aware {+ return plan_err!(+ "Cannot execute a null-aware join with two unbounded inputs: \+ SymmetricHashJoinExec does not support null-aware semantics."+ );+ }// Determine the partition mode based on configuration.let mode = if config_options.optimizer.repartition_joins {StreamJoinPartitionMode::Partitioneddiff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 8a6143bc68..3ef942035e 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -4741,5 +4741,104 @@ NULL NULL NULLstatement okDROP TABLE p26572;+# Regression for https://github.com/apache/datafusion/issues/27216: converting a+# null-aware NOT IN join to a symmetric hash join loses its NULL semantics.+query I+COPY (VALUES (1, 'x'), (2, 'x'), (CAST(NULL AS INT), 'x'))+TO 'test_files/scratch/subquery/not_in_left.csv'+STORED AS CSV OPTIONS ('format.has_header' 'true');+----+3++query I+COPY (VALUES (2, 'y'), (CAST(NULL AS INT), 'y'))+TO 'test_files/scratch/subquery/not_in_right.csv'+STORED AS CSV OPTIONS ('format.has_header' 'true');+----+2++statement ok+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_left(a INT, b VARCHAR)+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_left.csv'+OPTIONS ('format.has_header' 'true');++statement ok+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_right(a INT, b VARCHAR)+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_right.csv'+OPTIONS ('format.has_header' 'true');++# These null-aware joins cannot be executed as streaming symmetric hash joins.+# Reject them during planning instead of returning rows with incorrect NULL handling.+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics+SELECT a FROM unbounded_not_in_left+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right);++# NOT IN inside OR requires a null-aware LeftMark join.+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics+SELECT a FROM unbounded_not_in_left+WHERE a = 1 OR a NOT IN (SELECT a FROM unbounded_not_in_right);++# Nullable outer values still require null-aware semantics when the subquery excludes NULLs.+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics+SELECT a FROM unbounded_not_in_left+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right WHERE a IS NOT NULL);++# IN in a projection also requires a null-aware LeftMark join to preserve UNKNOWN.+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics+SELECT a, a IN (SELECT a FROM unbounded_not_in_right) AS is_present+FROM unbounded_not_in_left;++# Blocking subqueries retain the pipeline-breaking error from SanityCheckPlan.+query error Cannot execute pipeline breaking queries+SELECT a FROM unbounded_not_in_left+WHERE a NOT IN (SELECT max(a) FROM unbounded_not_in_right GROUP BY b);++query error Cannot execute pipeline breaking queries+SELECT a FROM unbounded_not_in_left+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right ORDER BY a LIMIT 5);++statement ok+DROP TABLE unbounded_not_in_left;++statement ok+DROP TABLE unbounded_not_in_right;++# With non-nullable keys, NOT IN still runs as a symmetric hash join.+query I+COPY (VALUES (1, 'x'), (2, 'x'))+TO 'test_files/scratch/subquery/not_in_left_nn.csv'+STORED AS CSV OPTIONS ('format.has_header' 'true');+----+2++query I+COPY (VALUES (2, 'y'))+TO 'test_files/scratch/subquery/not_in_right_nn.csv'+STORED AS CSV OPTIONS ('format.has_header' 'true');+----+1++statement ok+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_left_nn(a INT NOT NULL, b VARCHAR)+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_left_nn.csv'+OPTIONS ('format.has_header' 'true');++statement ok+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_right_nn(a INT NOT NULL, b VARCHAR)+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_right_nn.csv'+OPTIONS ('format.has_header' 'true');++query I+SELECT a FROM unbounded_not_in_left_nn+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right_nn);+----+1++statement ok+DROP TABLE unbounded_not_in_left_nn;++statement ok+DROP TABLE unbounded_not_in_right_nn;+statement okRESET datafusion.execution.batch_size;