Skip to content

Apache DataFusion SQL Query Engine

fix: reject unbounded null-aware joins (#27256)

Phoenixf94aaa187658
diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rs
index b8a3aefbe8..2f4eef0cbf 100644
--- a/datafusion/physical-optimizer/src/join_selection.rs
+++ b/datafusion/physical-optimizer/src/join_selection.rs
@@ -16,7 +16,7 @@ use datafusion_common::Statistics;
use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion_common::{JoinSide, JoinType, internal_err};
+use datafusion_common::{JoinSide, JoinType, internal_err, plan_err};
use datafusion_expr_common::sort_properties::SortProperties;
use datafusion_physical_expr::LexOrdering;
use datafusion_physical_expr::expressions::Column;
@@ -367,6 +367,11 @@ pub type PipelineFixerSubrule =
/// An `Option` that contains the `Result` of the transformation. If the transformation is not applicable,
/// it returns `None`. If applicable, it returns `Some(Ok(...))` with the modified pipeline state,
/// or `Some(Err(...))` if an error occurs during the transformation.
+///
+/// # Errors
+/// Returns a planning error if both inputs are unbounded and emit incrementally,
+/// and the join is null-aware. The symmetric hash join used for these inputs
+/// does not support null-aware semantics.
fn hash_join_convert_symmetric_subrule(
input: Arc<dyn ExecutionPlan>,
config_options: &ConfigOptions,
@@ -385,6 +390,12 @@ fn hash_join_convert_symmetric_subrule(
);
// Process only if both left and right sides are unbounded and incrementally emit.
if left_unbounded && right_unbounded & left_incremental & right_incremental {
+ if hash_join.null_aware {
+ return plan_err!(
+ "Cannot execute a null-aware join with two unbounded inputs: \
+ SymmetricHashJoinExec does not support null-aware semantics."
+ );
+ }
// Determine the partition mode based on configuration.
let mode = if config_options.optimizer.repartition_joins {
StreamJoinPartitionMode::Partitioned
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index 8a6143bc68..3ef942035e 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -4741,5 +4741,104 @@ NULL NULL NULL
statement ok
DROP TABLE p26572;
+# Regression for https://github.com/apache/datafusion/issues/27216: converting a
+# null-aware NOT IN join to a symmetric hash join loses its NULL semantics.
+query I
+COPY (VALUES (1, 'x'), (2, 'x'), (CAST(NULL AS INT), 'x'))
+TO 'test_files/scratch/subquery/not_in_left.csv'
+STORED AS CSV OPTIONS ('format.has_header' 'true');
+----
+3
+
+query I
+COPY (VALUES (2, 'y'), (CAST(NULL AS INT), 'y'))
+TO 'test_files/scratch/subquery/not_in_right.csv'
+STORED AS CSV OPTIONS ('format.has_header' 'true');
+----
+2
+
+statement ok
+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_left(a INT, b VARCHAR)
+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_left.csv'
+OPTIONS ('format.has_header' 'true');
+
+statement ok
+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_right(a INT, b VARCHAR)
+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_right.csv'
+OPTIONS ('format.has_header' 'true');
+
+# These null-aware joins cannot be executed as streaming symmetric hash joins.
+# Reject them during planning instead of returning rows with incorrect NULL handling.
+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics
+SELECT a FROM unbounded_not_in_left
+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right);
+
+# NOT IN inside OR requires a null-aware LeftMark join.
+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics
+SELECT a FROM unbounded_not_in_left
+WHERE a = 1 OR a NOT IN (SELECT a FROM unbounded_not_in_right);
+
+# Nullable outer values still require null-aware semantics when the subquery excludes NULLs.
+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics
+SELECT a FROM unbounded_not_in_left
+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right WHERE a IS NOT NULL);
+
+# IN in a projection also requires a null-aware LeftMark join to preserve UNKNOWN.
+query error Cannot execute a null-aware join with two unbounded inputs: SymmetricHashJoinExec does not support null-aware semantics
+SELECT a, a IN (SELECT a FROM unbounded_not_in_right) AS is_present
+FROM unbounded_not_in_left;
+
+# Blocking subqueries retain the pipeline-breaking error from SanityCheckPlan.
+query error Cannot execute pipeline breaking queries
+SELECT a FROM unbounded_not_in_left
+WHERE a NOT IN (SELECT max(a) FROM unbounded_not_in_right GROUP BY b);
+
+query error Cannot execute pipeline breaking queries
+SELECT a FROM unbounded_not_in_left
+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right ORDER BY a LIMIT 5);
+
+statement ok
+DROP TABLE unbounded_not_in_left;
+
+statement ok
+DROP TABLE unbounded_not_in_right;
+
+# With non-nullable keys, NOT IN still runs as a symmetric hash join.
+query I
+COPY (VALUES (1, 'x'), (2, 'x'))
+TO 'test_files/scratch/subquery/not_in_left_nn.csv'
+STORED AS CSV OPTIONS ('format.has_header' 'true');
+----
+2
+
+query I
+COPY (VALUES (2, 'y'))
+TO 'test_files/scratch/subquery/not_in_right_nn.csv'
+STORED AS CSV OPTIONS ('format.has_header' 'true');
+----
+1
+
+statement ok
+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_left_nn(a INT NOT NULL, b VARCHAR)
+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_left_nn.csv'
+OPTIONS ('format.has_header' 'true');
+
+statement ok
+CREATE UNBOUNDED EXTERNAL TABLE unbounded_not_in_right_nn(a INT NOT NULL, b VARCHAR)
+STORED AS CSV LOCATION 'test_files/scratch/subquery/not_in_right_nn.csv'
+OPTIONS ('format.has_header' 'true');
+
+query I
+SELECT a FROM unbounded_not_in_left_nn
+WHERE a NOT IN (SELECT a FROM unbounded_not_in_right_nn);
+----
+1
+
+statement ok
+DROP TABLE unbounded_not_in_left_nn;
+
+statement ok
+DROP TABLE unbounded_not_in_right_nn;
+
statement ok
RESET datafusion.execution.batch_size;