Skip to content

Apache DataFusion SQL Query Engine

feat: support multi-column NOT IN subqueries with null-aware joins (#19857)

Liang-Chi Hsieh532cd0376448
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q10.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q10.benchmark
new file mode 100644
index 0000000000..1857ef64d2
--- /dev/null
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q10.benchmark
@@ -0,0 +1,33 @@
+name Q10
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+# Correctness canary: the NOT IN result must match a reference count that does
+# not use NOT IN. It holds for every NAJ_ROWS. No key holds a NULL, so a row is
+# kept exactly when no subquery row equals its whole tuple.
+assert I
+SELECT count(*) = (
+ SELECT count(*)
+ FROM small_outer o
+ WHERE NOT EXISTS (
+ SELECT 1 FROM small_inner i WHERE i.id_n0 = o.id_n0 AND i.z = o.z
+ )
+)
+FROM small_outer o
+WHERE (o.id_n0, o.z) NOT IN (SELECT i.id_n0, i.z FROM small_inner i);
+----
+true
+
+expect_plan HashJoinExec
+
+run
+-- Q10: uncorrelated multi-column NOT IN over nullable keys that hold no NULL.
+-- A tuple NOT IN always decides UNKNOWN per build row (the outer side), but
+-- without NULLs there is no candidate pair to check, so this is the floor for
+-- the multi-column shape.
+SELECT count(*)
+FROM small_outer o
+WHERE (o.id_n0, o.z) NOT IN (SELECT i.id_n0, i.z FROM small_inner i);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q11.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q11.benchmark
new file mode 100644
index 0000000000..067616db49
--- /dev/null
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q11.benchmark
@@ -0,0 +1,35 @@
+name Q11
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+# Correctness canary: the NOT IN result must match a reference count that does
+# not use NOT IN. It holds for every NAJ_ROWS. The outer tuple holds no NULL, so
+# a subquery row makes it UNKNOWN or TRUE exactly when `z` matches and `id_n1`
+# is NULL or equal; the row is kept only when no such subquery row exists.
+assert I
+SELECT count(*) = (
+ SELECT count(*)
+ FROM small_outer o
+ WHERE NOT EXISTS (
+ SELECT 1 FROM small_inner i
+ WHERE i.z = o.z AND (i.id_n1 IS NULL OR i.id_n1 = o.id_n0)
+ )
+)
+FROM small_outer o
+WHERE (o.id_n0, o.z) NOT IN (SELECT i.id_n1, i.z FROM small_inner i);
+----
+true
+
+expect_plan HashJoinExec
+
+run
+-- Q11: uncorrelated multi-column NOT IN with 1% NULL in one subquery element.
+-- Each NULL-valued subquery row is paired with every outer row and the pair is
+-- kept as UNKNOWN only when the other element matches, so the cost grows with
+-- (outer rows) x (NULL-valued subquery rows). Compare with Q10 and Q12.
+SELECT count(*)
+FROM small_outer o
+WHERE (o.id_n0, o.z) NOT IN (SELECT i.id_n1, i.z FROM small_inner i);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark
new file mode 100644
index 0000000000..c126c9c161
--- /dev/null
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark
@@ -0,0 +1,37 @@
+name Q12
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+# Correctness canary: the NOT IN result must match a reference count that does
+# not use NOT IN. It holds for every NAJ_ROWS. The subquery holds no NULL, so a
+# subquery row makes an outer tuple UNKNOWN or TRUE exactly when `z` matches and
+# the outer `id_n1` is NULL or equal; the row is kept only when no such
+# subquery row exists.
+assert I
+SELECT count(*) = (
+ SELECT count(*)
+ FROM small_outer o
+ WHERE NOT EXISTS (
+ SELECT 1 FROM small_inner i
+ WHERE i.z = o.z AND (o.id_n1 IS NULL OR i.id_n0 = o.id_n1)
+ )
+)
+FROM small_outer o
+WHERE (o.id_n1, o.z) NOT IN (SELECT i.id_n0, i.z FROM small_inner i);
+----
+true
+
+expect_plan HashJoinExec
+
+run
+-- Q12: uncorrelated multi-column NOT IN with 1% NULL in one outer element,
+-- the mirror of Q11. The outer side is the build side, so each NULL-valued
+-- build row is paired with every subquery row and kept as UNKNOWN only when
+-- the other element matches: the cost grows with (NULL-valued outer rows) x
+-- (subquery rows). Compare with Q10 and Q11.
+SELECT count(*)
+FROM small_outer o
+WHERE (o.id_n1, o.z) NOT IN (SELECT i.id_n0, i.z FROM small_inner i);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/null_aware_join/init/load.sql b/benchmarks/sql_benchmarks/null_aware_join/init/load.sql
index 1225c71995..379a4f9bd3 100644
--- a/benchmarks/sql_benchmarks/null_aware_join/init/load.sql
+++ b/benchmarks/sql_benchmarks/null_aware_join/init/load.sql
@@ -14,7 +14,7 @@
-- `id` is only ~half covered by the subquery side, so the anti join returns rows
-- rather than degenerating to an empty or full result.
--- Small tables: used by the correlated queries (Q04-Q08), whose cost grows with
+-- Small tables: used by the correlated and multi-column queries (Q04-Q12), whose cost grows with
-- the product of the two table sizes.
CREATE TABLE small_outer AS
SELECT
diff --git a/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite b/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite
index d827c7ed58..470f65a95f 100644
--- a/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite
+++ b/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite
@@ -1,4 +1,4 @@
-description = "Null-aware (NOT IN) hash join benchmarks: uncorrelated, non-equality-correlated, and equality-correlated NOT IN across NULL fractions, plus the positive IN shape that must not be null-aware"
+description = "Null-aware (NOT IN) hash join benchmarks: uncorrelated, non-equality-correlated, and equality-correlated NOT IN across NULL fractions, plus the positive IN shape that must not be null-aware, and multi-column (tuple) NOT IN with and without NULLs"
query_pattern = "q{QUERY_ID_PADDED}.benchmark"
@@ -8,7 +8,7 @@ short = "r"
env = "NAJ_ROWS"
default = "10000"
values = ["10000", "..."]
-help = "Rows per table for the correlated queries (Q04-Q09). These evaluate the join filter over candidate build x probe pairs, so their cost grows with the square of this value."
+help = "Rows per table for the correlated and multi-column queries (Q04-Q12). These evaluate the join filter over candidate build x probe pairs, so their cost grows with the square of this value."
[[options]]
name = "large-rows"
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index f11b39157c..b48bc19a55 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -41,7 +41,7 @@ use crate::physical_plan::explain::ExplainExec;
use crate::physical_plan::filter::FilterExecBuilder;
use crate::physical_plan::joins::utils as join_utils;
use crate::physical_plan::joins::{
- AsOfJoinExec, AsOfMatchExpr, CrossJoinExec, HashJoinExec, NestedLoopJoinExec,
+ AsOfJoinExec, AsOfMatchExpr, CrossJoinExec, HashJoinExecBuilder, NestedLoopJoinExec,
PartitionMode, SortMergeJoinExec,
};
use crate::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
@@ -1358,6 +1358,7 @@ impl DefaultPhysicalPlanner {
join_type,
null_equality,
null_aware,
+ null_aware_value_keys,
schema: join_schema,
..
}) => {
@@ -1790,17 +1791,20 @@ impl DefaultPhysicalPlanner {
PartitionMode::CollectLeft
};
- Arc::new(HashJoinExec::try_new(
- physical_left,
- physical_right,
- join_on,
- join_filter,
- join_type,
- None,
- partition_mode,
- *null_equality,
- *null_aware,
- )?)
+ Arc::new(
+ HashJoinExecBuilder::new(
+ physical_left,
+ physical_right,
+ join_on,
+ *join_type,
+ )
+ .with_filter(join_filter)
+ .with_partition_mode(partition_mode)
+ .with_null_equality(*null_equality)
+ .with_null_aware(*null_aware)
+ .with_null_aware_value_keys(*null_aware_value_keys)
+ .build()?,
+ )
};
// If plan was mutated previously then need to create the ExecutionPlan
diff --git a/datafusion/core/tests/fuzz_cases/mod.rs b/datafusion/core/tests/fuzz_cases/mod.rs
index 198151465b..563c48dc67 100644
--- a/datafusion/core/tests/fuzz_cases/mod.rs
+++ b/datafusion/core/tests/fuzz_cases/mod.rs
@@ -21,6 +21,7 @@ mod distinct_count_string_fuzz;
#[expect(clippy::needless_pass_by_value)]
mod join_fuzz;
mod merge_fuzz;
+mod null_aware_join_fuzz;
#[expect(clippy::needless_pass_by_value)]
mod sort_fuzz;
#[expect(clippy::needless_pass_by_value)]
diff --git a/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs b/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs
new file mode 100644
index 0000000000..4bf7171771
--- /dev/null
+++ b/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Fuzz test for null-aware (`NOT IN`) joins.
+//!
+//! Scalar and multi-column `NOT IN` / `IN` subqueries, which plan as
+//! null-aware hash joins, are compared with `NOT EXISTS` / `EXISTS`
+//! formulations of the same three-valued logic, which do not. Random tables
+//! with NULLs on both sides are swept across batch sizes and partition counts.
+
+use std::sync::Arc;
+
+use arrow::array::{Int32Array, RecordBatch};
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::util::pretty::pretty_format_batches;
+use datafusion::datasource::MemTable;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+
+#[tokio::test(flavor = "multi_thread")]
+async fn null_aware_join_matches_reference() {
+ for seed in 0..20 {
+ let mut rng = StdRng::seed_from_u64(seed);
+ let outer = random_table(&mut rng, &["id", "k", "a", "b", "c"]);
+ let inner = random_table(&mut rng, &["k", "x", "y", "z"]);
+ for target_partitions in [1, 4] {
+ for batch_size in [1, 3, 8192] {
+ let config = SessionConfig::new()
+ .with_target_partitions(target_partitions)
+ .with_batch_size(batch_size)
+ // Keep multi-partition runs from collapsing to a single
+ // partition on these small inputs.
+ .set_usize(
+ "datafusion.optimizer.hash_join_single_partition_threshold",
+ 0,
+ )
+ .set_usize(
+ "datafusion.optimizer.hash_join_single_partition_threshold_rows",
+ 0,
+ );
+ let ctx = SessionContext::new_with_config(config);
+ ctx.register_table("o", Arc::clone(&outer) as _).unwrap();
+ ctx.register_table("i", Arc::clone(&inner) as _).unwrap();
+ for (query, reference) in cases() {
+ let actual = run(&ctx, &query).await;
+ let expected = run(&ctx, &reference).await;
+ assert_eq!(
+ actual, expected,
+ "seed={seed} target_partitions={target_partitions} \
+ batch_size={batch_size}\nquery: {query}\nreference: {reference}"
+ );
+ }
+ }
+ }
+ }
+}
+
+/// Pairs of (query, reference) that must return the same rows.
+fn cases() -> Vec<(String, String)> {
+ let mut cases = vec![];
+ // (correlation inside the subquery, the same correlation in the reference)
+ for (correlation, reference_correlation) in [
+ ("", ""),
+ (" WHERE i.k = o.k", " AND i.k = o.k"),
+ (" WHERE i.k < o.k", " AND i.k < o.k"),
+ (
+ " WHERE i.k = o.k AND i.z < o.c",
+ " AND i.k = o.k AND i.z < o.c",
+ ),
+ ] {
+ // (outer value, subquery output, element-wise equality)
+ for (value, output, equal) in [
+ ("o.a", "i.x", "o.a = i.x"),
+ ("(o.a, o.b)", "i.x, i.y", "o.a = i.x AND o.b = i.y"),
+ (
+ "(o.a, o.b, o.c)",
+ "i.x, i.y, i.z",
+ "o.a = i.x AND o.b = i.y AND o.c = i.z",
+ ),
+ ("(o.a, 1)", "i.x, i.y", "o.a = i.x AND 1 = i.y"),
+ ] {
+ let subquery = format!("SELECT {output} FROM i{correlation}");
+ let matched = format!(
+ "EXISTS (SELECT 1 FROM i WHERE ({equal}){reference_correlation})"
+ );
+ let unknown = format!(
+ "EXISTS (SELECT 1 FROM i WHERE ({equal}) IS NULL{reference_correlation})"
+ );
+ let not_in = format!(
+ "NOT EXISTS (SELECT 1 FROM i WHERE ({equal}) IS NOT FALSE{reference_correlation})"
+ );
+ cases.push((
+ format!("SELECT o.id FROM o WHERE {value} NOT IN ({subquery})"),
+ format!("SELECT o.id FROM o WHERE {not_in}"),
+ ));
+ cases.push((
+ format!("SELECT o.id FROM o WHERE {value} IN ({subquery})"),
+ format!("SELECT o.id FROM o WHERE {matched}"),
+ ));
+ cases.push((
+ format!(
+ "SELECT o.id FROM o WHERE {value} NOT IN ({subquery}) OR o.id % 3 = 0"
+ ),
+ format!("SELECT o.id FROM o WHERE {not_in} OR o.id % 3 = 0"),
+ ));
+ cases.push((
+ format!("SELECT o.id, {value} NOT IN ({subquery}) AS r FROM o"),
+ format!(
+ "SELECT o.id, CASE WHEN {matched} THEN false \
+ WHEN {unknown} THEN NULL ELSE true END AS r FROM o"
+ ),
+ ));
+ }
+ }
+ cases
+}
+
+/// A table of nullable `Int32` columns with values in `0..3`, split into up
+/// to three partitions. A column named `id` holds unique row ids instead.
+fn random_table(rng: &mut StdRng, columns: &[&str]) -> Arc<MemTable> {
+ let schema = Arc::new(Schema::new(
+ columns
+ .iter()
+ .map(|name| Field::new(*name, DataType::Int32, true))
+ .collect::<Vec<_>>(),
+ ));
+ let null_fraction = [0.0, 0.1, 0.3][rng.random_range(0..3)];
+ let mut next_id = 0;
+ let partitions = (0..rng.random_range(1..4))
+ .map(|_| {
+ let rows = rng.random_range(0..6);
+ let arrays = columns
+ .iter()
+ .map(|name| {
+ let values: Int32Array = if *name == "id" {
+ (next_id..next_id + rows).map(Some).collect()
+ } else {
+ (0..rows)
+ .map(|_| {
+ (!rng.random_bool(null_fraction))
+ .then(|| rng.random_range(0..3))
+ })
+ .collect()
+ };
+ Arc::new(values) as _
+ })
+ .collect();
+ next_id += rows;
+ vec![RecordBatch::try_new(Arc::clone(&schema), arrays).unwrap()]
+ })
+ .collect();
+ Arc::new(MemTable::try_new(schema, partitions).unwrap())
+}
+
+/// The rows of `sql`, sorted.
+async fn run(ctx: &SessionContext, sql: &str) -> Vec<String> {
+ let batches = ctx.sql(sql).await.unwrap().collect().await.unwrap();
+ let formatted = pretty_format_batches(&batches).unwrap().to_string();
+ let mut rows: Vec<String> = formatted
+ .lines()
+ .filter(|line| line.starts_with('|'))
+ .skip(1)
+ .map(String::from)
+ .collect();
+ rows.sort();
+ rows
+}
diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs
index c197a6af6f..30fcf47f58 100644
--- a/datafusion/expr/src/expr.rs
+++ b/datafusion/expr/src/expr.rs
@@ -17,6 +17,7 @@
//! Logical Expressions: [`Expr`]
+use std::borrow::Cow;
use std::cmp::Ordering;
use std::collections::HashSet;
use std::fmt::{self, Display, Formatter, Write};
@@ -1401,6 +1402,54 @@ impl InSubquery {
}
}
+/// The tuple elements of a multi-column `(a, b, ...) IN (SELECT x, y, ...)`,
+/// given the compared expression `expr` and the `subquery` plan, or `None`
+/// for a single-column `IN`.
+///
+/// The tuple is planned as a `struct` call, which `simplify_expressions` folds
+/// into a struct literal when every element is a literal. It is a
+/// multi-column `IN` only when the subquery returns more than one column;
+/// against a single column the struct is one value compared with a
+/// struct-typed column.
+///
+/// Errors when the number of tuple elements does not match the number of
+/// subquery columns.
+pub fn in_subquery_tuple_values<'a>(
+ expr: &'a Expr,
+ subquery: &crate::LogicalPlan,
+) -> Result<Option<Cow<'a, [Expr]>>> {
+ let num_columns = subquery.schema().fields().len();
+ if num_columns <= 1 {
+ return Ok(None);
+ }
+ let values = match expr {
+ Expr::ScalarFunction(func) if func.func.name() == "struct" => {
+ Cow::Borrowed(func.args.as_slice())
+ }
+ Expr::Literal(ScalarValue::Struct(array), _)
+ if arrow::array::Array::is_valid(array.as_ref(), 0) =>
+ {
+ Cow::Owned(
+ array
+ .columns()
+ .iter()
+ .map(|column| {
+ Ok(Expr::Literal(ScalarValue::try_from_array(column, 0)?, None))
+ })
+ .collect::<Result<_>>()?,
+ )
+ }
+ _ => return Ok(None),
+ };
+ if values.len() != num_columns {
+ return plan_err!(
+ "The number of columns in the tuple ({}) must match the number of columns in the subquery ({num_columns})",
+ values.len()
+ );
+ }
+ Ok(Some(values))
+}
+
/// Placeholder, representing bind parameter values such as `$1` or `$name`.
///
/// The type of these parameters is inferred using [`Expr::infer_placeholder_types`]
@@ -2224,11 +2273,32 @@ impl Expr {
subquery,
negated: _,
}) => {
- rewrite_placeholder_from_subquery(
- "InSubquery",
- expr.as_mut(),
- subquery,
- )?;
+ // Multi-column `(a, b) IN (SELECT x, y ...)`: infer each
+ // tuple element from the subquery column at the same
+ // position.
+ let subquery_schema = subquery.subquery.schema();
+ let is_tuple =
+ in_subquery_tuple_values(expr.as_ref(), &subquery.subquery)?
+ .is_some();
+ match expr.as_mut() {
+ Expr::ScalarFunction(func) if is_tuple => {
+ for (i, arg) in func.args.iter_mut().enumerate() {
+ // Qualified, since two subquery columns can
+ // share a name under different qualifiers.
+ let column = Expr::Column(Column::from(
+ subquery_schema.qualified_field(i),
+ ));
+ rewrite_placeholder(arg, &column, subquery_schema)?;
+ }
+ }
+ // A struct literal holds no placeholder.
+ _ if is_tuple => {}
+ expr => rewrite_placeholder_from_subquery(
+ "InSubquery",
+ expr,
+ subquery,
+ )?,
+ }
}
Expr::SetComparison(SetComparison {
expr,
diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs
index c47b45612a..8aaca2b72b 100644
--- a/datafusion/expr/src/expr_schema.rs
+++ b/datafusion/expr/src/expr_schema.rs
@@ -20,7 +20,7 @@ use crate::ValueOrLambda;
use crate::expr::{
AggregateFunction, AggregateFunctionParams, Alias, BinaryExpr, Cast, InList,
InSubquery, Lambda, Placeholder, ScalarFunction, TryCast, Unnest, WindowFunction,
- WindowFunctionParams,
+ WindowFunctionParams, in_subquery_tuple_values,
};
use crate::expr::{FieldMetadata, LambdaVariable};
use crate::higher_order_function::HigherOrderReturnFieldArgs;
@@ -408,6 +408,23 @@ impl ExprSchemable for Expr {
| Expr::Exists { .. } => Ok(false),
Expr::SetComparison(_) => Ok(true),
Expr::InSubquery(InSubquery { expr, subquery, .. }) => {
+ // A multi-column `(a, b) IN (SELECT x, y ...)` is UNKNOWN when
+ // any element or subquery column is NULL. The tuple is a
+ // `struct` call, which itself is never NULL, so its elements
+ // and every subquery column count.
+ if let Some(values) = in_subquery_tuple_values(expr, &subquery.subquery)?
+ {
+ let mut nullable = subquery
+ .subquery
+ .schema()
+ .fields()
+ .iter()
+ .any(|field| field.is_nullable());
+ for value in values.iter() {
+ nullable |= value.nullable(input_schema)?;
+ }
+ return Ok(nullable);
+ }
let expr_nullable = expr.nullable(input_schema)?;
let subquery_nullable = subquery.subquery.schema().fields().first().ok_or_else(|| {
plan_datafusion_err!("subquery must return exactly one column of data to compare against")
@@ -818,27 +835,58 @@ fn scalar_subquery_nullable(subquery: &Subquery) -> bool {
/// 2. **Non-projection plan**: If the subquery isn't a projection, it adds a projection to the plan
/// with the casted first column.
pub fn cast_subquery(subquery: Subquery, cast_to_type: &DataType) -> Result<Subquery> {
- if subquery.subquery.schema().field(0).data_type() == cast_to_type {
+ cast_subquery_columns(subquery, std::slice::from_ref(cast_to_type))
+}
+
+/// Cast the leading columns of a subquery to the given types, one per column,
+/// like [`cast_subquery`] does for its first column. The result keeps only
+/// the `cast_to_types.len()` leading columns.
+///
+/// Used by a multi-column `(a, b) IN (SELECT x, y ...)`, whose tuple elements
+/// are coerced column by column.
+pub fn cast_subquery_columns(
+ subquery: Subquery,
+ cast_to_types: &[DataType],
+) -> Result<Subquery> {
+ let schema = subquery.subquery.schema();
+ if schema
+ .fields()
+ .iter()
+ .zip(cast_to_types)
+ .all(|(field, cast_to_type)| field.data_type() == cast_to_type)
+ {
return Ok(subquery);
}
let plan = subquery.subquery.as_ref();
let new_plan = match plan {
LogicalPlan::Projection(projection) => {
- let cast_expr = projection.expr[0]
- .clone()
- .cast_to(cast_to_type, projection.input.schema())?;
+ let cast_exprs = projection
+ .expr
+ .iter()
+ .zip(cast_to_types)
+ .map(|(expr, cast_to_type)| {
+ expr.clone()
+ .cast_to(cast_to_type, projection.input.schema())
+ })
+ .collect::<Result<Vec<_>>>()?;
LogicalPlan::Projection(Projection::try_new(
- vec![cast_expr],
+ cast_exprs,
Arc::clone(&projection.input),
)?)
}
_ => {
- let cast_expr = Expr::Column(Column::from(plan.schema().qualified_field(0)))
- .cast_to(cast_to_type, subquery.subquery.schema())?;
+ let cast_exprs = cast_to_types
+ .iter()
+ .enumerate()
+ .map(|(i, cast_to_type)| {
+ Expr::Column(Column::from(plan.schema().qualified_field(i)))
+ .cast_to(cast_to_type, subquery.subquery.schema())
+ })
+ .collect::<Result<Vec<_>>>()?;
LogicalPlan::Projection(Projection::try_new(
- vec![cast_expr],
- subquery.subquery,
+ cast_exprs,
+ Arc::clone(&subquery.subquery),
)?)
}
};
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index 86a8dfa8db..798b9ea3e9 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -1282,6 +1282,7 @@ impl LogicalPlanBuilder {
schema: DFSchemaRef::new(join_schema),
null_equality,
null_aware,
+ null_aware_value_keys: 1,
})))
}
diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rs
index 95438c94d4..375c7dfe08 100644
--- a/datafusion/expr/src/logical_plan/invariants.rs
+++ b/datafusion/expr/src/logical_plan/invariants.rs
@@ -22,7 +22,7 @@ use datafusion_common::{
use crate::{
Aggregate, DmlStatement, Expr, Filter, Join, JoinType, LogicalPlan, Window, WriteOp,
- expr::{Exists, InSubquery, SetComparison},
+ expr::{Exists, InSubquery, SetComparison, in_subquery_tuple_values},
expr_rewriter::strip_outer_reference,
utils::{collect_subquery_cols, split_conjunction},
};
@@ -224,13 +224,20 @@ pub fn check_subquery_expr(
}
} else {
if let Expr::InSubquery(subquery) = expr {
- // InSubquery should only return one column
- if subquery.subquery.subquery.schema().fields().len() > 1 {
- return plan_err!(
- "InSubquery should only return one column, but found {}: {}",
- subquery.subquery.subquery.schema().fields().len(),
- subquery.subquery.subquery.schema().field_names().join(", ")
- );
+ // InSubquery should only return one column, unless it is a
+ // multi-column `(a, b) IN (SELECT x, y ...)` with one tuple element
+ // per subquery column.
+ let num_subquery_cols = subquery.subquery.subquery.schema().fields().len();
+ match in_subquery_tuple_values(&subquery.expr, &subquery.subquery.subquery)? {
+ Some(_) => {}
+ None if num_subquery_cols > 1 => {
+ return plan_err!(
+ "InSubquery should only return one column, but found {}: {}",
+ num_subquery_cols,
+ subquery.subquery.subquery.schema().field_names().join(", ")
+ );
+ }
+ None => {}
}
}
if let Expr::SetComparison(set_comparison) = expr
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index ba6b1a6a26..658516843a 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -688,6 +688,7 @@ impl LogicalPlan {
schema: _,
null_equality,
null_aware,
+ null_aware_value_keys,
}) => {
let schema =
build_join_schema(left.schema(), right.schema(), &join_type)?;
@@ -710,6 +711,7 @@ impl LogicalPlan {
schema: DFSchemaRef::new(schema),
null_equality,
null_aware,
+ null_aware_value_keys,
}))
}
LogicalPlan::AsOfJoin(AsOfJoin {
@@ -991,6 +993,7 @@ impl LogicalPlan {
on,
null_equality,
null_aware,
+ null_aware_value_keys,
..
}) => {
let (left, right) = self.only_two_inputs(inputs)?;
@@ -1033,6 +1036,7 @@ impl LogicalPlan {
schema: DFSchemaRef::new(schema),
null_equality: *null_equality,
null_aware: *null_aware,
+ null_aware_value_keys: *null_aware_value_keys,
}))
}
LogicalPlan::AsOfJoin(AsOfJoin {
@@ -2258,6 +2262,7 @@ impl LogicalPlan {
join_constraint,
join_type,
null_aware,
+ null_aware_value_keys,
..
}) => {
let join_expr: Vec<String> =
@@ -2266,8 +2271,12 @@ impl LogicalPlan {
.as_ref()
.map(|expr| format!(" Filter: {expr}"))
.unwrap_or_else(|| "".to_string());
- let null_aware_expr =
- if *null_aware { " null_aware" } else { "" };
+ let null_aware_expr = match (*null_aware, *null_aware_value_keys)
+ {
+ (false, _) => "".to_string(),
+ (true, 0 | 1) => " null_aware".to_string(),
+ (true, n) => format!(" null_aware(value_keys={n})"),
+ };
let join_type = if filter.is_none()
&& keys.is_empty()
&& *join_type == JoinType::Inner
@@ -4480,6 +4489,17 @@ pub struct Join {
/// For `LeftMark`, the generated `mark` column becomes nullable so unmatched rows can produce
/// `NULL` rather than `false` when SQL three-valued logic requires it.
pub null_aware: bool,
+ /// Number of `NOT IN` value keys of a null-aware join; ignored unless
+ /// [`Self::null_aware`] is set.
+ ///
+ /// A scalar `x NOT IN (SELECT y ...)` has one value key, a multi-column
+ /// `(a, b) NOT IN (SELECT x, y ...)` one per tuple element. The value keys
+ /// are the leading equi-join keys (`on[..n]`, or the leading equality
+ /// conjuncts of `filter` before they are extracted into `on`); any keys
+ /// after them are correlation scope keys of a correlated subquery. A NULL
+ /// in a value key follows SQL three-valued logic across the whole tuple,
+ /// while the scope keys only select which subquery rows are compared.
+ pub null_aware_value_keys: usize,
}
/// The ordered comparison used by an [`AsOfJoin`].
@@ -4707,9 +4727,17 @@ impl Join {
schema: Arc::new(join_schema),
null_equality,
null_aware,
+ null_aware_value_keys: 1,
})
}
+ /// Sets the number of `NOT IN` value keys of a null-aware join, see
+ /// [`Self::null_aware_value_keys`].
+ pub fn with_null_aware_value_keys(mut self, null_aware_value_keys: usize) -> Self {
+ self.null_aware_value_keys = null_aware_value_keys;
+ self
+ }
+
/// Create Join with input which wrapped with projection, this method is used in physical planning only to help
/// create the physical join.
pub fn try_new_with_project_input(
@@ -4762,6 +4790,7 @@ impl Join {
schema: Arc::new(join_schema),
null_equality: original_join.null_equality,
null_aware: original_join.null_aware,
+ null_aware_value_keys: original_join.null_aware_value_keys,
},
requalified,
))
@@ -6928,6 +6957,7 @@ mod tests {
schema: Arc::new(left_schema.join(&right_schema)?),
null_equality: NullEquality::NullEqualsNothing,
null_aware: false,
+ null_aware_value_keys: 1,
}))
}
diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rs
index ab796e88d7..9e961a1f11 100644
--- a/datafusion/expr/src/logical_plan/tree_node.rs
+++ b/datafusion/expr/src/logical_plan/tree_node.rs
@@ -138,6 +138,7 @@ impl TreeNode for LogicalPlan {
schema,
null_equality,
null_aware,
+ null_aware_value_keys,
}) => (left, right).map_elements(f)?.update_data(|(left, right)| {
LogicalPlan::Join(Join {
left,
@@ -149,6 +150,7 @@ impl TreeNode for LogicalPlan {
schema,
null_equality,
null_aware,
+ null_aware_value_keys,
})
}),
LogicalPlan::AsOfJoin(AsOfJoin {
@@ -628,6 +630,7 @@ impl LogicalPlan {
schema,
null_equality,
null_aware,
+ null_aware_value_keys,
}) => (on, filter).map_elements(f)?.update_data(|(on, filter)| {
LogicalPlan::Join(Join {
left,
@@ -639,6 +642,7 @@ impl LogicalPlan {
schema,
null_equality,
null_aware,
+ null_aware_value_keys,
})
}),
LogicalPlan::AsOfJoin(AsOfJoin {
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index c09552be0a..e805e5eeb2 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -37,10 +37,10 @@ use datafusion_common::{
use datafusion_expr::expr::{
self, AggregateFunctionParams, Alias, Between, BinaryExpr, Case, Exists,
HigherOrderFunction, InList, InSubquery, Like, ScalarFunction, SetComparison, Sort,
- WindowFunction,
+ WindowFunction, in_subquery_tuple_values,
};
use datafusion_expr::expr_rewriter::coerce_plan_expr_for_schema;
-use datafusion_expr::expr_schema::cast_subquery;
+use datafusion_expr::expr_schema::{cast_subquery, cast_subquery_columns};
use datafusion_expr::logical_plan::Subquery;
use datafusion_expr::type_coercion::binary::{
comparison_coercion_with_session_timezone, like_coercion, regex_coercion,
@@ -648,26 +648,49 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> {
self.session_time_zone,
)?
.data;
- let expr_type = expr.get_type(self.schema)?;
- let subquery_type = new_plan.schema().field(0).data_type();
- let common_type = comparison_coercion_with_session_timezone(
- &expr_type,
- subquery_type,
- self.session_time_zone,
- )
- .ok_or(plan_datafusion_err!(
- "expr type {expr_type} can't cast to {subquery_type} in InSubquery"
- ))?;
let new_subquery = Subquery {
subquery: Arc::new(new_plan),
outer_ref_columns: subquery.outer_ref_columns,
spans: subquery.spans,
};
- Ok(Transformed::yes(Expr::InSubquery(InSubquery::new(
- Box::new(expr.cast_to(&common_type, self.schema)?),
- cast_subquery(new_subquery, &common_type)?,
- negated,
- ))))
+ let is_tuple =
+ in_subquery_tuple_values(&expr, &new_subquery.subquery)?.is_some();
+ match *expr {
+ // Multi-column `(a, b) IN (SELECT x, y ...)`: the tuple is
+ // planned as a `struct` call whose arguments are compared
+ // pairwise with the subquery columns, so coerce each pair.
+ Expr::ScalarFunction(func) if is_tuple => {
+ let (expr, subquery) = coerce_multi_column_in_subquery(
+ func,
+ new_subquery,
+ self.schema,
+ self.session_time_zone,
+ )?;
+ Ok(Transformed::yes(Expr::InSubquery(InSubquery::new(
+ Box::new(expr),
+ subquery,
+ negated,
+ ))))
+ }
+ expr => {
+ let expr_type = expr.get_type(self.schema)?;
+ let subquery_type =
+ new_subquery.subquery.schema().field(0).data_type();
+ let common_type = comparison_coercion_with_session_timezone(
+ &expr_type,
+ subquery_type,
+ self.session_time_zone,
+ )
+ .ok_or(plan_datafusion_err!(
+ "expr type {expr_type} can't cast to {subquery_type} in InSubquery"
+ ))?;
+ Ok(Transformed::yes(Expr::InSubquery(InSubquery::new(
+ Box::new(expr.cast_to(&common_type, self.schema)?),
+ cast_subquery(new_subquery, &common_type)?,
+ negated,
+ ))))
+ }
+ }
}
Expr::SetComparison(SetComparison {
expr,
@@ -1677,6 +1700,40 @@ fn project_with_column_index(
.map(LogicalPlan::Projection)
}
+/// Coerces a multi-column `(a, b, ...) IN (SELECT x, y, ...)`.
+///
+/// Each tuple element and the subquery column at the same position are cast
+/// to their common comparison type, like the single-column form does for its
+/// one pair.
+fn coerce_multi_column_in_subquery(
+ func: ScalarFunction,
+ subquery: Subquery,
+ schema: &DFSchema,
+ session_time_zone: Option<&str>,
+) -> Result<(Expr, Subquery)> {
+ let subquery_schema = Arc::clone(subquery.subquery.schema());
+ let mut common_types = Vec::with_capacity(func.args.len());
+ let mut args = Vec::with_capacity(func.args.len());
+ for (arg, field) in func.args.into_iter().zip(subquery_schema.fields()) {
+ let expr_type = arg.get_type(schema)?;
+ let subquery_type = field.data_type();
+ let common_type = comparison_coercion_with_session_timezone(
+ &expr_type,
+ subquery_type,
+ session_time_zone,
+ )
+ .ok_or(plan_datafusion_err!(
+ "expr type {expr_type} can't cast to {subquery_type} in InSubquery"
+ ))?;
+ args.push(arg.cast_to(&common_type, schema)?);
+ common_types.push(common_type);
+ }
+ Ok((
+ Expr::ScalarFunction(ScalarFunction::new_udf(func.func, args)),
+ cast_subquery_columns(subquery, &common_types)?,
+ ))
+}
+
#[cfg(test)]
mod test {
diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
index 34f446f5e8..bfd5109761 100644
--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
@@ -29,9 +29,10 @@ use crate::{OptimizerConfig, OptimizerRule};
use datafusion_common::alias::AliasGenerator;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::{
- Column, DFSchema, NullEquality, Result, ScalarValue, assert_or_internal_err, plan_err,
+ Column, DFSchema, NullEquality, Result, ScalarValue, assert_or_internal_err,
+ internal_datafusion_err, internal_err, not_impl_err, plan_err,
};
-use datafusion_expr::expr::{Exists, InSubquery};
+use datafusion_expr::expr::{Exists, InSubquery, in_subquery_tuple_values};
use datafusion_expr::expr_rewriter::{
create_col_from_scalar_expr, strip_outer_reference,
};
@@ -351,6 +352,14 @@ fn in_subquery_value_mark_join(
)));
}
+ // The fallback below compares the `IN` value as a single value, which a
+ // multi-column `(a, b) IN (SELECT x, y ...)` is not. `build_join` rejects
+ // the tuples whose mark cannot be exact, so this is not reached for them.
+ if in_subquery_tuple_values(&expr, subquery)?.is_some() {
+ return internal_err!(
+ "a multi-column IN subquery needs a three-valued exact mark join"
+ );
+ }
// SQL IN needs three facts per outer row to distinguish FALSE from UNKNOWN.
let null_subquery = LogicalPlanBuilder::from(subquery.clone())
.filter(output_expr.is_null())?
@@ -557,11 +566,12 @@ struct MarkJoin {
three_valued_exact: bool,
}
-/// Is the `IN` equality the first equality of `join_filter` that the hash
-/// join can use as a key? A null-aware hash join reads its first key as the
-/// `IN` value and the other keys as the correlation.
-fn value_is_first_key(
- in_value: &InValue,
+/// Are the `IN` equalities, in order, the first equalities of `join_filter`
+/// that the hash join can use as keys? A null-aware hash join reads its first
+/// `V` keys as the `IN` values (one per tuple element of a multi-column
+/// `IN`) and the other keys as the correlation.
+fn values_are_first_keys(
+ in_values: &[InValue],
join_filter: &Expr,
left_schema: &DFSchema,
right_schema: &DFSchema,
@@ -571,12 +581,17 @@ fn value_is_first_key(
left_schema,
right_schema,
)?;
- Ok(equijoin_keys.first().is_some_and(|(value, column)| {
- value == &in_value.value && column == &in_value.subquery_column
- }))
+ Ok(equijoin_keys.len() >= in_values.len()
+ && equijoin_keys
+ .iter()
+ .zip(in_values)
+ .all(|((value, column), in_value)| {
+ value == &in_value.value && column == &in_value.subquery_column
+ }))
}
-/// The two sides of an `IN` predicate, `value IN (SELECT output_expr ..)`.
+/// The two sides of an `IN` predicate, `value IN (SELECT output_expr ..)`, or
+/// of one tuple element of a multi-column `(a, b) IN (SELECT x, y ..)`.
struct InValue {
/// The value from the outer plan, as the join filter refers to it. This is
/// a projected column when `value_as_written` is a constant.
@@ -704,18 +719,51 @@ fn strip_casts(expr: &Expr) -> &Expr {
expr
}
-/// Combines the `IN`/`NOT IN` equality with the correlation predicates, if any.
+/// Combines the `IN`/`NOT IN` equalities, one per value, with the correlation
+/// predicates, if any.
///
-/// The equality stays the leading conjunct so that it becomes `on[0]`, the key
-/// position a null-aware hash join reads as the `NOT IN` value key (see
-/// `HashJoinExec::null_aware`).
-fn in_predicate_first(in_predicate: Expr, correlation: Option<Expr>) -> Expr {
- match correlation {
- Some(correlation) => in_predicate.and(correlation),
- None => in_predicate,
+/// The equalities stay the leading conjuncts, in order, so that they become
+/// `on[..V]`, the key positions a null-aware hash join reads as the `NOT IN`
+/// value keys (see `HashJoinExec::null_aware`).
+fn in_join_filter(in_values: &[InValue], correlation: Option<Expr>) -> Result<Expr> {
+ conjunction(
+ in_values
+ .iter()
+ .map(|in_value| {
+ Expr::eq(in_value.value.clone(), in_value.subquery_column.clone())
+ })
+ .chain(correlation),
+ )
+ .ok_or_else(|| internal_datafusion_err!("an `IN` predicate has at least one value"))
+}
+
+/// The `i`-th output expression of `subquery`, as the query writes it.
+fn subquery_output_expr(subquery: &LogicalPlan, i: usize) -> Result<Expr> {
+ match subquery {
+ _ if i == 0 => subquery
+ .head_output_expr()?
+ .ok_or_else(|| internal_datafusion_err!("subquery has no output expression")),
+ LogicalPlan::Projection(projection) => Ok(projection.expr[i].clone()),
+ _ => Ok(Expr::Column(Column::from(
+ subquery.schema().qualified_field(i),
+ ))),
}
}
+/// Sets [`Join::null_aware_value_keys`](datafusion_expr::Join::null_aware_value_keys)
+/// on a join built by [`build_join`].
+fn with_null_aware_value_keys(
+ plan: LogicalPlan,
+ null_aware_value_keys: usize,
+) -> Result<LogicalPlan> {
+ let LogicalPlan::Join(join) = plan else {
+ return internal_err!("expected a join, got {}", plan.display());
+ };
+ Ok(LogicalPlan::Join(
+ join.with_null_aware_value_keys(null_aware_value_keys),
+ ))
+}
+
/// The outcome of [`build_join`].
struct BuiltJoin {
/// The outer plan with the subquery joined into it.
@@ -758,35 +806,55 @@ fn build_join(
})?;
// The two sides of the `IN` predicate: the value from the outer plan and
- // the subquery output it is compared with, renamed to the alias.
- let in_value = match in_predicate_opt {
+ // the subquery output it is compared with, renamed to the alias. A
+ // multi-column `(a, b) IN (SELECT x, y ...)` has one pair per tuple
+ // element; its `in_predicate` compares the whole tuple with the first
+ // subquery output, so each element is paired with its own output here.
+ // `EXISTS` has none.
+ let in_values = match in_predicate_opt {
Some(Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::Eq,
right,
})) => {
- let right_col = create_col_from_scalar_expr(right, alias.to_string())?;
- Some(InValue {
- value: left.deref().clone(),
- subquery_column: Expr::Column(right_col),
- value_as_written: left.deref().clone(),
- output_expr: right.deref().clone(),
- })
+ let values = match in_subquery_tuple_values(left, subquery)? {
+ Some(values) => values.into_owned(),
+ None => vec![left.deref().clone()],
+ };
+ values
+ .into_iter()
+ .enumerate()
+ .map(|(i, value)| {
+ let output_expr = if i == 0 {
+ right.deref().clone()
+ } else {
+ subquery_output_expr(subquery, i)?
+ };
+ let right_col =
+ create_col_from_scalar_expr(&output_expr, alias.to_string())?;
+ Ok(InValue {
+ value: value.clone(),
+ subquery_column: Expr::Column(right_col),
+ value_as_written: value,
+ output_expr,
+ })
+ })
+ .collect::<Result<Vec<_>>>()?
}
Some(_) => return Ok(None),
- None => None,
+ None => vec![],
};
- // True if the `IN` value or the subquery column it is compared with can be
- // NULL inside the scope of an outer row. `EXISTS` has no value to compare.
- let value_may_be_null = match &in_value {
- Some(in_value) => in_value.may_be_null_in_scope(
+ // True if an `IN` value or the subquery column it is compared with can be
+ // NULL inside the scope of an outer row.
+ let mut value_may_be_null = false;
+ for in_value in &in_values {
+ value_may_be_null |= in_value.may_be_null_in_scope(
left.schema(),
sub_query_alias.schema(),
&pull_up.correlated_filters,
- )?,
- None => false,
- };
+ )?;
+ }
// Whether this join needs null-aware (`NOT IN` three-valued) semantics.
// Decided once: the constant projection below, the mark branch and the anti
@@ -816,57 +884,77 @@ fn build_join(
// Projecting the constant as a column of the outer side turns the predicate
// into a real equi-join key, which fixes all three. It only pays for itself
// on a join that ends up null-aware.
+ //
+ // Every constant element of a tuple is projected the same way, so that all
+ // value keys lead the equi-join keys in order.
let mut projected_left = None;
- let in_value = match in_value {
- Some(in_value) if in_value.value.column_refs().is_empty() && null_aware => {
+ let mut in_values = in_values;
+ if null_aware
+ && in_values
+ .iter()
+ .any(|in_value| in_value.value.column_refs().is_empty())
+ {
+ let left_schema = left.schema();
+ let mut projections = left_schema
+ .columns()
+ .into_iter()
+ .map(Expr::from)
+ .collect::<Vec<_>>();
+ let is_tuple = in_values.len() > 1;
+ for (i, in_value) in in_values.iter_mut().enumerate() {
+ if !in_value.value.column_refs().is_empty() {
+ continue;
+ }
// The projected column is unqualified, so a left field that already
// has this name — however unlikely — would make the reference
// ambiguous.
- let mut value_name = format!("{alias}_value");
- let left_schema = left.schema();
+ let mut value_name = if is_tuple {
+ format!("{alias}_value_{i}")
+ } else {
+ format!("{alias}_value")
+ };
while left_schema.fields().iter().any(|f| f.name() == &value_name) {
value_name.push('_');
}
let value_col = Column::new_unqualified(value_name);
- let projections = left_schema
- .columns()
- .into_iter()
- .map(Expr::from)
- .chain(std::iter::once(in_value.value.alias(value_col.name())))
- .collect::<Vec<_>>();
- projected_left = Some(
- LogicalPlanBuilder::from(left.clone())
- .project(projections)?
- .build()?,
- );
- Some(InValue {
- value: Expr::Column(value_col),
- ..in_value
- })
+ let constant =
+ std::mem::replace(&mut in_value.value, Expr::Column(value_col.clone()));
+ projections.push(constant.alias(value_col.name()));
}
- other => other,
- };
+ projected_left = Some(
+ LogicalPlanBuilder::from(left.clone())
+ .project(projections)?
+ .build()?,
+ );
+ }
let left = projected_left.as_ref().unwrap_or(left);
- let join_filter = match (&in_value, join_filter_opt) {
- (Some(in_value), correlation_opt) => in_predicate_first(
- Expr::eq(in_value.value.clone(), in_value.subquery_column.clone()),
- correlation_opt,
- ),
- (None, Some(correlation)) => correlation,
- (None, None) => lit(true),
+ let join_filter = match (in_values.is_empty(), join_filter_opt) {
+ (false, correlation_opt) => in_join_filter(&in_values, correlation_opt)?,
+ (true, Some(correlation)) => correlation,
+ (true, None) => lit(true),
};
// Only a null-aware join reads the key order.
- let value_is_first_key = match &in_value {
- Some(in_value) if null_aware => value_is_first_key(
- in_value,
+ let value_is_first_key = null_aware
+ && values_are_first_keys(
+ &in_values,
&join_filter,
left.schema(),
sub_query_alias.schema(),
- )?,
- _ => false,
- };
+ )?;
+
+ // A multi-column `IN` has no fallback that materializes the UNKNOWN rows
+ // (see below), so a tuple element that does not become its key, for
+ // example because its type is not hashable, is rejected instead of being
+ // read in the wrong key position.
+ if null_aware && !value_is_first_key && in_values.len() > 1 {
+ return not_impl_err!(
+ "Multi-column NOT IN subquery is not supported when a tuple element is not a hashable equi-join key: {join_filter}"
+ );
+ }
+ // The first `V` keys of a null-aware join are its `NOT IN` value keys.
+ let null_aware_value_keys = in_values.len().max(1);
// A `NOT IN` in a filter builds a `LeftAnti` join. The null-aware hash
// join reads its first key as the `NOT IN` value and the other keys as the
@@ -916,19 +1004,22 @@ fn build_join(
// nullable mark column. A non-equality correlation stays behind as a
// join filter, which the hash join also applies when it decides
// whether a NULL makes the mark UNKNOWN.
- let mark_is_three_valued_exact = in_value.is_some()
+ let mark_is_three_valued_exact = !in_values.is_empty()
&& (!value_may_be_null || (null_aware && value_is_first_key));
- let new_plan = LogicalPlanBuilder::from(left.clone())
- .join_detailed_with_options(
- right_projected,
- join_type,
- (Vec::<Column>::new(), Vec::<Column>::new()),
- Some(join_filter),
- NullEquality::NullEqualsNothing,
- null_aware,
- )?
- .build()?;
+ let new_plan = with_null_aware_value_keys(
+ LogicalPlanBuilder::from(left.clone())
+ .join_detailed_with_options(
+ right_projected,
+ join_type,
+ (Vec::<Column>::new(), Vec::<Column>::new()),
+ Some(join_filter),
+ NullEquality::NullEqualsNothing,
+ null_aware,
+ )?
+ .build()?,
+ null_aware_value_keys,
+ )?;
debug!(
"predicate subquery optimized:\n{}",
@@ -944,16 +1035,19 @@ fn build_join(
// join our sub query into the main plan
let new_plan = if null_aware {
// Use join_detailed_with_options to set null_aware flag
- LogicalPlanBuilder::from(left.clone())
- .join_detailed_with_options(
- sub_query_alias,
- join_type,
- (Vec::<Column>::new(), Vec::<Column>::new()), // No equijoin keys, filter-based join
- Some(join_filter),
- NullEquality::NullEqualsNothing,
- true, // null_aware
- )?
- .build()?
+ with_null_aware_value_keys(
+ LogicalPlanBuilder::from(left.clone())
+ .join_detailed_with_options(
+ sub_query_alias,
+ join_type,
+ (Vec::<Column>::new(), Vec::<Column>::new()), // No equijoin keys, filter-based join
+ Some(join_filter),
+ NullEquality::NullEqualsNothing,
+ true, // null_aware
+ )?
+ .build()?,
+ null_aware_value_keys,
+ )?
} else {
LogicalPlanBuilder::from(left.clone())
.join_on(sub_query_alias, join_type, Some(join_filter))?
diff --git a/datafusion/optimizer/src/eliminate_cross_join.rs b/datafusion/optimizer/src/eliminate_cross_join.rs
index bdc70da593..d1e72e254e 100644
--- a/datafusion/optimizer/src/eliminate_cross_join.rs
+++ b/datafusion/optimizer/src/eliminate_cross_join.rs
@@ -351,6 +351,7 @@ fn find_inner_join(
schema: join_schema,
null_equality,
null_aware: false,
+ null_aware_value_keys: 1,
}));
}
}
@@ -374,6 +375,7 @@ fn find_inner_join(
join_constraint: JoinConstraint::On,
null_equality,
null_aware: false,
+ null_aware_value_keys: 1,
}))
}
@@ -1409,6 +1411,7 @@ mod tests {
schema: join_schema,
null_equality: NullEquality::NullEqualsNull, // Test preservation
null_aware: false,
+ null_aware_value_keys: 1,
});
// Apply filter that can create join conditions
diff --git a/datafusion/optimizer/src/eliminate_join.rs b/datafusion/optimizer/src/eliminate_join.rs
index 25151fd484..8215ae7bec 100644
--- a/datafusion/optimizer/src/eliminate_join.rs
+++ b/datafusion/optimizer/src/eliminate_join.rs
@@ -510,16 +510,19 @@ fn rewrite_join(
if changed {
// The join type or an input changed, so the output schema may have
// narrowed; recompute it via `try_new`.
- Ok(Transformed::yes(LogicalPlan::Join(Join::try_new(
- left,
- right,
- join.on,
- join.filter,
- rewritten_join_type,
- join.join_constraint,
- join.null_equality,
- join.null_aware,
- )?)))
+ Ok(Transformed::yes(LogicalPlan::Join(
+ Join::try_new(
+ left,
+ right,
+ join.on,
+ join.filter,
+ rewritten_join_type,
+ join.join_constraint,
+ join.null_equality,
+ join.null_aware,
+ )?
+ .with_null_aware_value_keys(join.null_aware_value_keys),
+ )))
} else {
// Nothing changed; reassemble the join reusing its existing schema rather
// than recomputing it.
@@ -533,6 +536,7 @@ fn rewrite_join(
schema: join.schema,
null_equality: join.null_equality,
null_aware: join.null_aware,
+ null_aware_value_keys: join.null_aware_value_keys,
})))
}
}
diff --git a/datafusion/optimizer/src/eliminate_outer_join.rs b/datafusion/optimizer/src/eliminate_outer_join.rs
index a327da66f8..466c5bb83a 100644
--- a/datafusion/optimizer/src/eliminate_outer_join.rs
+++ b/datafusion/optimizer/src/eliminate_outer_join.rs
@@ -181,6 +181,7 @@ fn try_simplify_join(join: &Join, predicate: &Expr) -> Option<LogicalPlan> {
schema: Arc::clone(&join.schema),
null_equality: join.null_equality,
null_aware: join.null_aware,
+ null_aware_value_keys: join.null_aware_value_keys,
}))
}
diff --git a/datafusion/optimizer/src/extract_equijoin_predicate.rs b/datafusion/optimizer/src/extract_equijoin_predicate.rs
index 03a5aaa204..af5e601ef9 100644
--- a/datafusion/optimizer/src/extract_equijoin_predicate.rs
+++ b/datafusion/optimizer/src/extract_equijoin_predicate.rs
@@ -77,6 +77,7 @@ impl OptimizerRule for ExtractEquijoinPredicate {
schema,
null_equality,
null_aware,
+ null_aware_value_keys,
}) => {
let left_schema = left.schema();
let right_schema = right.schema();
@@ -119,6 +120,7 @@ impl OptimizerRule for ExtractEquijoinPredicate {
// safe to override it
null_equality: NullEquality::NullEqualsNull,
null_aware,
+ null_aware_value_keys,
})));
}
}
@@ -135,6 +137,7 @@ impl OptimizerRule for ExtractEquijoinPredicate {
schema,
null_equality,
null_aware,
+ null_aware_value_keys,
})))
} else {
Ok(Transformed::no(LogicalPlan::Join(Join {
@@ -147,6 +150,7 @@ impl OptimizerRule for ExtractEquijoinPredicate {
schema,
null_equality,
null_aware,
+ null_aware_value_keys,
})))
}
}
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 10a9fa70a8..f6ff292556 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -70,7 +70,8 @@ use crate::{
metrics::{ExecutionPlanMetricsSet, MetricsSet},
};
-use arrow::array::{Array, ArrayRef, BooleanBufferBuilder, UInt64Array};
+use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, UInt64Array};
+use arrow::buffer::NullBuffer;
use arrow::compute::concat_batches;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
@@ -201,8 +202,9 @@ fn array_map_key_range(
Ok(Some((min_val, max_val)))
}
-/// The build rows whose scalar `NOT IN` value key is NULL, used by correlated
-/// null-aware joins (see [`NullAwareMode`]).
+/// The build rows with a NULL `NOT IN` value key (in any element of a
+/// multi-column key), used by correlated null-aware joins (see
+/// [`NullAwareMode`]).
///
/// Such rows are UNKNOWN whenever *any* probe row in their correlation scope
/// passes the join filter, so every probe row must be tested against them.
@@ -222,27 +224,40 @@ pub(super) struct NullValueBuildRows {
pub(super) build_indices: UInt64Array,
}
+/// Returns the rows where at least one of `value_keys` is NULL, or `None` when
+/// there is no such row.
+pub(super) fn null_value_key_mask(value_keys: &[ArrayRef]) -> Option<BooleanArray> {
+ let logical_nulls: Vec<_> = value_keys.iter().map(|v| v.logical_nulls()).collect();
+ let valid = NullBuffer::union_many(logical_nulls.iter().map(Option::as_ref))?;
+ (valid.null_count() > 0).then(|| BooleanArray::new(!valid.inner(), None))
+}
+
/// Null-aware (`NOT IN`) semantics of a hash join, derived from
/// [`HashJoinExec::null_aware`] and the join type.
///
/// Only these combinations are legal (see [`Self::try_new`]), so the
/// stream matches on this instead of re-checking `null_aware && join_type == ..`.
///
-/// A `correlated` join has correlation scope keys (`on[1..]`, see
-/// [`HashJoinExec::null_aware`]) or a join filter, or both. A NULL then makes
-/// `NOT IN` UNKNOWN only for the build rows whose scope and filter keep that
-/// NULL, so the join records the decision per build row in the null-indices
-/// bitmap instead of in shared probe-side flags.
+/// A `correlated` join has more than one key — correlation scope keys
+/// (`on[V..]`, see [`HashJoinExec::null_aware`]), a multi-column value key, or
+/// both — or a join filter. A NULL then makes `NOT IN` UNKNOWN only for the
+/// build rows whose scope, filter and other tuple elements keep that NULL, so
+/// the join records the decision per build row in the null-indices bitmap
+/// instead of in shared probe-side flags.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum NullAwareMode {
/// `build.key NOT IN (probe.key)`: emits build rows. When uncorrelated,
/// none of them are emitted once any probe key is NULL.
- LeftAnti { correlated: bool },
+ ///
+ /// `value_keys` is the number of `NOT IN` value keys, see
+ /// [`HashJoinExec::null_aware_value_keys`].
+ LeftAnti { correlated: bool, value_keys: usize },
/// Uncorrelated `probe.key NOT IN (build.key)`: emits probe rows, and
/// none of them once any build key is NULL.
RightAnti,
- /// `NOT IN` as a nullable mark column on the build rows.
- LeftMark { correlated: bool },
+ /// `NOT IN` as a nullable mark column on the build rows; `value_keys` as
+ /// for `LeftAnti`.
+ LeftMark { correlated: bool, value_keys: usize },
}
impl NullAwareMode {
@@ -251,14 +266,30 @@ impl NullAwareMode {
join_type: JoinType,
partition_mode: PartitionMode,
num_keys: usize,
+ num_value_keys: usize,
has_filter: bool,
) -> Result<Self> {
+ if num_value_keys == 0 || num_value_keys > num_keys {
+ return plan_err!(
+ "null_aware {join_type} join needs between 1 and {num_keys} `NOT IN` value keys, got {num_value_keys}"
+ );
+ }
+ // `num_keys > 1` also covers a multi-column value key without any
+ // correlation: a NULL in one tuple element leaves the comparison FALSE
+ // whenever another element is a definite mismatch, so it too must be
+ // decided per build row.
let correlated = num_keys > 1 || has_filter;
let mode = match (join_type, partition_mode) {
- (JoinType::LeftAnti, _) => Self::LeftAnti { correlated },
+ (JoinType::LeftAnti, _) => Self::LeftAnti {
+ correlated,
+ value_keys: num_value_keys,
+ },
// `PartitionMode::CollectLeft` is safe because `RightAnti` is probe-driven
(JoinType::RightAnti, PartitionMode::CollectLeft) => Self::RightAnti,
- (JoinType::LeftMark, _) => Self::LeftMark { correlated },
+ (JoinType::LeftMark, _) => Self::LeftMark {
+ correlated,
+ value_keys: num_value_keys,
+ },
_ => {
return plan_err!(
"null_aware can only be true for LeftAnti joins and RightAnti joins with `CollectLeft` `PartitionMode`, or LeftMark joins, got {join_type} with {partition_mode}"
@@ -271,9 +302,10 @@ impl NullAwareMode {
),
// Correlated joins share the per-build-row null bitmap across all
// probe partitions.
- Self::LeftMark { .. } | Self::LeftAnti { correlated: true }
- if partition_mode == PartitionMode::Partitioned =>
- {
+ Self::LeftMark { .. }
+ | Self::LeftAnti {
+ correlated: true, ..
+ } if partition_mode == PartitionMode::Partitioned => {
plan_err!(
"null_aware joins require PartitionMode::CollectLeft, got PartitionMode::Partitioned"
)
@@ -289,9 +321,26 @@ impl NullAwareMode {
pub(super) fn is_correlated(self) -> bool {
matches!(
self,
- Self::LeftAnti { correlated: true } | Self::LeftMark { correlated: true }
+ Self::LeftAnti {
+ correlated: true,
+ ..
+ } | Self::LeftMark {
+ correlated: true,
+ ..
+ }
)
}
+
+ /// Number of leading keys that are `NOT IN` value keys; the keys after
+ /// them are correlation scope keys.
+ pub(super) fn value_keys(self) -> usize {
+ match self {
+ Self::LeftAnti { value_keys, .. } | Self::LeftMark { value_keys, .. } => {
+ value_keys
+ }
+ Self::RightAnti => 1,
+ }
+ }
}
/// Immutable build buffers and their durable reservation.
@@ -318,12 +367,12 @@ struct JoinBuildData {
pub(super) struct JoinLeftData {
/// Hash table over correlated scope keys for correlated null-aware joins.
///
- /// Key 0 is the scalar `NOT IN` value key and keys 1..N are correlated
- /// equality scope keys. This map covers all build rows and is probed only
+ /// Keys `..V` are the `NOT IN` value keys and keys `V..` the correlated
+ /// equality scope keys, where `V` is `null_aware_value_keys`. This map covers all build rows and is probed only
/// with NULL-valued probe rows; the complementary direction uses
/// `null_value_build_rows`. `None` when there are no scope keys.
null_aware_scope_map: Option<Box<dyn JoinHashMapType>>,
- /// The build rows whose value key is NULL (see [`NullValueBuildRows`]).
+ /// The build rows with a NULL value key (see [`NullValueBuildRows`]).
/// `None` when the build side has no NULL value keys.
null_value_build_rows: Option<NullValueBuildRows>,
/// Shared bitmap for visited left indices within this consuming join.
@@ -469,6 +518,7 @@ impl HashJoinExecBuilder {
column_indices: vec![],
null_equality: NullEquality::NullEqualsNothing,
null_aware: false,
+ null_aware_value_keys: 1,
dynamic_filter: None,
// Will be computed at when plan will be built.
cache: stub_properties(),
@@ -531,6 +581,13 @@ impl HashJoinExecBuilder {
self
}
+ /// Set the number of `NOT IN` value keys of a null-aware join, see
+ /// [`HashJoinExec::null_aware_value_keys`].
+ pub fn with_null_aware_value_keys(mut self, null_aware_value_keys: usize) -> Self {
+ self.exec.null_aware_value_keys = null_aware_value_keys;
+ self
+ }
+
/// Set fetch property.
pub fn with_fetch(mut self, fetch: Option<usize>) -> Self {
self.exec.fetch = fetch;
@@ -614,6 +671,7 @@ impl HashJoinExecBuilder {
projection,
null_equality,
null_aware,
+ null_aware_value_keys,
dynamic_filter,
fetch,
// Recomputed.
@@ -663,6 +721,7 @@ impl HashJoinExecBuilder {
column_indices,
null_equality,
null_aware,
+ null_aware_value_keys,
cache: Arc::new(cache),
dynamic_filter,
fetch,
@@ -694,6 +753,7 @@ impl From<&HashJoinExec> for HashJoinExecBuilder {
column_indices: exec.column_indices.clone(),
null_equality: exec.null_equality,
null_aware: exec.null_aware,
+ null_aware_value_keys: exec.null_aware_value_keys,
cache: Arc::clone(&exec.cache),
dynamic_filter: exec.dynamic_filter.clone(),
fetch: exec.fetch,
@@ -940,13 +1000,31 @@ pub struct HashJoinExec {
/// only the probe rows that pass it take part in the three-valued logic.
///
/// Key-ordering convention (relied on positionally, not enforced): for a
- /// null-aware `LeftAnti` or `LeftMark` join with more than one key, `on[0]`
- /// is the scalar `NOT IN` value key and `on[1..N]` are the correlated
- /// equality scope keys.
+ /// null-aware `LeftAnti` or `LeftMark` join, `on[..V]` are the `NOT IN`
+ /// value keys and `on[V..]` the correlated equality scope keys, where `V`
+ /// is [`Self::null_aware_value_keys`].
/// Reordering these keys would silently produce wrong results, which is why
/// such joins are pinned to `PartitionMode::CollectLeft` (the only key
/// reorderer acts solely on `PartitionMode::Partitioned`).
pub null_aware: bool,
+ /// Number of `NOT IN` value keys of a null-aware join; ignored unless
+ /// [`Self::null_aware`] is set.
+ ///
+ /// A scalar `x NOT IN (SELECT y ...)` has one value key; a multi-column
+ /// `(a, b) NOT IN (SELECT x, y ...)` has one per tuple element. The tuple
+ /// comparison is UNKNOWN rather than FALSE when no element pair is a
+ /// definite mismatch and at least one of them involves a NULL, so a NULL in
+ /// one element does not make every comparison UNKNOWN the way a scalar
+ /// NULL does.
+ ///
+ /// Such a join always decides UNKNOWN per build row, like a correlated
+ /// scalar `NOT IN`, which has two costs the uncorrelated scalar join does
+ /// not: the outer side stays the build side, since only a single-key join
+ /// is swapped, and without correlation keys every NULL-valued row is paired
+ /// with every row on the other side to check its other elements. See
+ /// <https://github.com/apache/datafusion/issues/26088> and
+ /// <https://github.com/apache/datafusion/issues/26089> for the follow-ups.
+ pub null_aware_value_keys: usize,
/// Cache holding plan properties like equivalences, output partitioning etc.
cache: Arc<PlanProperties>,
/// Dynamic filter for pushing down to the probe side
@@ -980,6 +1058,7 @@ impl fmt::Debug for HashJoinExec {
.field("random_state", &self.random_state)
.field("mode", &self.mode)
.field("null_aware", &self.null_aware)
+ .field("null_aware_value_keys", &self.null_aware_value_keys)
.field("metrics", &self.metrics)
.field("projection", &self.projection)
.field("column_indices", &self.column_indices)
@@ -1290,6 +1369,7 @@ impl HashJoinExec {
self.join_type,
self.mode,
self.on.len(),
+ self.null_aware_value_keys,
self.filter.is_some(),
)
})
@@ -1494,7 +1574,11 @@ impl DisplayAs for HashJoinExec {
.fetch
.map_or_else(String::new, |f| format!(", fetch={f}"));
let display_null_aware =
- if self.null_aware { ", null_aware" } else { "" };
+ match (self.null_aware, self.null_aware_value_keys) {
+ (false, _) => String::new(),
+ (true, 1) => ", null_aware".to_string(),
+ (true, n) => format!(", null_aware(value_keys={n})"),
+ };
let display_prepared = self
.prepared_build
.as_ref()
@@ -1546,7 +1630,15 @@ impl DisplayAs for HashJoinExec {
}
if self.null_aware {
- writeln!(f, "null_aware")?;
+ if self.null_aware_value_keys == 1 {
+ writeln!(f, "null_aware")?;
+ } else {
+ writeln!(
+ f,
+ "null_aware(value_keys={})",
+ self.null_aware_value_keys
+ )?;
+ }
}
if let Some(filter) = self.filter.as_ref() {
@@ -1722,6 +1814,8 @@ impl ExecutionPlan for HashJoinExec {
let join_metrics = BuildProbeJoinMetrics::new(partition, &self.metrics);
+ let null_aware = self.null_aware_mode()?;
+
let array_map_created_count = MetricBuilder::new(&self.metrics)
.with_category(MetricCategory::Rows)
.counter(ARRAY_MAP_CREATED_COUNT_METRIC_NAME, partition);
@@ -1747,7 +1841,7 @@ impl ExecutionPlan for HashJoinExec {
on_right,
repartition_random_state,
self.null_equality,
- self.null_aware,
+ null_aware,
))
})))
})
@@ -1755,8 +1849,6 @@ impl ExecutionPlan for HashJoinExec {
.flatten()
.flatten();
- let null_aware = self.null_aware_mode()?;
-
let left_fut = match (&self.prepared_build, self.mode) {
(Some(prepared), _) => {
// Public join fields can change after builder validation.
@@ -2138,6 +2230,7 @@ impl ExecutionPlan for HashJoinExec {
&self,
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
+ use datafusion_common::internal_datafusion_err;
use datafusion_proto_models::protobuf;
// Destructure exhaustively (no `..`) so that a newly added field is a
@@ -2152,6 +2245,7 @@ impl ExecutionPlan for HashJoinExec {
projection,
null_equality,
null_aware,
+ null_aware_value_keys,
dynamic_filter,
fetch,
// derived from the children's schemas by the builder on decode
@@ -2236,6 +2330,12 @@ impl ExecutionPlan for HashJoinExec {
Some(v) => v.iter().map(|x| *x as u32).collect(),
},
null_aware: *null_aware,
+ null_aware_value_keys: u32::try_from(*null_aware_value_keys)
+ .map_err(|_| {
+ internal_datafusion_err!(
+ "HashJoinExec: null_aware_value_keys {null_aware_value_keys} does not fit in u32"
+ )
+ })?,
dynamic_filter,
fetch: fetch.map(|f| f as u64),
},
@@ -2274,6 +2374,7 @@ impl HashJoinExec {
filter,
projection,
null_aware,
+ null_aware_value_keys,
dynamic_filter,
fetch,
} = &**hashjoin;
@@ -2347,6 +2448,16 @@ impl HashJoinExec {
.with_partition_mode(partition_mode)
.with_null_equality(null_equality)
.with_null_aware(*null_aware)
+ // Messages predating the field decode it as 0; they can only hold
+ // scalar `NOT IN` joins.
+ .with_null_aware_value_keys(
+ usize_from_wire(
+ u64::from(*null_aware_value_keys),
+ "HashJoinExec",
+ "null_aware_value_keys",
+ )?
+ .max(1),
+ )
.with_fetch(fetch)
.build()?;
@@ -2440,6 +2551,7 @@ mod proto_tests {
filter: None,
projection: vec![],
null_aware: false,
+ null_aware_value_keys: 0,
dynamic_filter: None,
fetch: None,
}
@@ -2612,6 +2724,39 @@ mod proto_tests {
assert!(as_join(&decode(node)).null_aware);
}
+ /// A multi-column `NOT IN` keeps its value key count through the round
+ /// trip; a node written before the field existed decodes it as 0, which
+ /// is read as a scalar `NOT IN`.
+ #[test]
+ fn null_aware_value_keys_round_trip() {
+ let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = vec![
+ (Arc::new(Column::new("a", 0)), Arc::new(Column::new("a", 0))),
+ (Arc::new(Column::new("a", 0)), Arc::new(Column::new("a", 0))),
+ ];
+ let plan =
+ HashJoinExecBuilder::new(stub_child(), stub_child(), on, JoinType::LeftAnti)
+ .with_partition_mode(PartitionMode::CollectLeft)
+ .with_null_aware(true)
+ .with_null_aware_value_keys(2)
+ .build()
+ .unwrap();
+ assert_eq!(
+ encode(&plan, &StubPlanEncoder::ok()).null_aware_value_keys,
+ 2
+ );
+
+ let mut node = decodable_node();
+ node.join_type = protobuf::JoinType::Leftanti.into();
+ node.partition_mode = protobuf::PartitionMode::CollectLeft.into();
+ node.null_aware = true;
+ node.on = vec![node.on[0].clone(), node.on[0].clone()];
+ node.null_aware_value_keys = 2;
+ assert_eq!(as_join(&decode(node.clone())).null_aware_value_keys, 2);
+
+ node.null_aware_value_keys = 0;
+ assert_eq!(as_join(&decode(node)).null_aware_value_keys, 1);
+ }
+
#[test]
fn try_to_proto_encodes_both_children_and_both_key_sides() {
let encoder = StubPlanEncoder::ok();
@@ -3058,6 +3203,7 @@ async fn collect_left_input(
// The extra scope maps + null bitmap are only built for correlated
// null-aware joins (see `NullAwareMode`).
let with_null_aware_row_state = null_aware.is_some_and(NullAwareMode::is_correlated);
+ let null_aware_value_keys = null_aware.map_or(1, NullAwareMode::value_keys);
let is_phj_candidate = is_perfect_hash_join_candidate(&on_left, &schema)?;
@@ -3258,9 +3404,9 @@ async fn collect_left_input(
};
let (null_aware_scope_map, null_value_build_rows) = if with_null_aware_row_state {
- // Null-aware convention: `on_left[0]` is the value key and
- // `on_left[1..]` the (possibly empty) correlation scope keys.
- let scope_keys = &on_left[1..];
+ // Null-aware convention: `on_left[..V]` are the value keys and
+ // `on_left[V..]` the (possibly empty) correlation scope keys.
+ let scope_keys = &on_left[null_aware_value_keys..];
let scope_map = if scope_keys.is_empty() {
None
} else {
@@ -3285,13 +3431,13 @@ async fn collect_left_input(
};
// Collect the NULL-valued build rows (see `NullValueBuildRows`).
- let value_key = &left_values[0];
- let null_value_build_rows = if value_key.logical_null_count() > 0 {
- let null_mask = arrow::compute::is_null(value_key.as_ref())?;
+ let null_value_build_rows = if let Some(null_mask) =
+ null_value_key_mask(&left_values[..null_aware_value_keys])
+ {
let build_indices = UInt64Array::from_iter_values(
null_mask.values().set_indices().map(|i| i as u64),
);
- let scope_values = normalized_values[1..]
+ let scope_values = normalized_values[null_aware_value_keys..]
.iter()
.map(|values| Ok(arrow::compute::filter(values.as_ref(), &null_mask)?))
.collect::<Result<Vec<_>>>()?;
@@ -10452,6 +10598,39 @@ mod tests {
Ok(())
}
+ /// The value key count must name at least one key and no more than the
+ /// join has.
+ #[tokio::test]
+ async fn test_null_aware_validation_value_keys() -> Result<()> {
+ for value_keys in [0, 3] {
+ let left = build_table_two_cols(("a", &vec![Some(1)]), ("b", &vec![Some(2)]));
+ let right =
+ build_table_two_cols(("x", &vec![Some(1)]), ("y", &vec![Some(2)]));
+ let on = vec![
+ (
+ Arc::new(Column::new_with_schema("a", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("x", &right.schema())?) as _,
+ ),
+ (
+ Arc::new(Column::new_with_schema("b", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("y", &right.schema())?) as _,
+ ),
+ ];
+ let err = HashJoinExecBuilder::new(left, right, on, JoinType::LeftAnti)
+ .with_partition_mode(PartitionMode::CollectLeft)
+ .with_null_aware(true)
+ .with_null_aware_value_keys(value_keys)
+ .build()
+ .unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("needs between 1 and 2 `NOT IN` value keys"),
+ "{err}"
+ );
+ }
+ Ok(())
+ }
+
/// Correlated null-aware `LeftAnti` with a two-column correlation scope,
/// so scope lookups compare keys with the general multi-column comparator.
///
diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs
index b6187f1090..7d81dc9162 100644
--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs
@@ -26,7 +26,7 @@ use crate::ExecutionPlanProperties;
use crate::Partitioning;
use crate::joins::Map;
use crate::joins::PartitionMode;
-use crate::joins::hash_join::exec::HASH_JOIN_SEED;
+use crate::joins::hash_join::exec::{HASH_JOIN_SEED, NullAwareMode};
use crate::joins::hash_join::inlist_builder::build_struct_fields;
use crate::joins::hash_join::partitioned_hash_eval::{
HashExpr, HashTableLookupExpr, SeededRandomState,
@@ -273,7 +273,7 @@ pub(crate) struct SharedBuildAccumulator {
null_equality: NullEquality,
/// Null-aware anti join (`NOT IN`). A probe-side NULL must reach the join so its
/// three-valued logic can collapse the result, so the pushed filter keeps NULL rows.
- null_aware: bool,
+ null_aware: Option<NullAwareMode>,
}
/// Strategy for filter pushdown (decided at collection time)
@@ -385,7 +385,7 @@ impl SharedBuildAccumulator {
on_right: Vec<PhysicalExprRef>,
repartition_random_state: SeededRandomState,
null_equality: NullEquality,
- null_aware: bool,
+ null_aware: Option<NullAwareMode>,
) -> Self {
// Troubleshooting: If partition counts are incorrect, verify this logic matches
// the actual execution pattern in collect_build_side()
@@ -809,8 +809,9 @@ impl SharedBuildAccumulator {
/// needs a NULL value key to reach the join for `NOT IN` three-valued logic, and a
/// null-equal join needs NULL keys to match build-side NULLs.
///
- /// For a null-aware join, only `on_right[0]` needs the escape: it is the scalar
- /// `NOT IN` value key, while `on_right[1..]` are correlation scope keys. For a
+ /// For a null-aware join, only the `NOT IN` value keys `on_right[..V]` need the
+ /// escape, while `on_right[V..]` are correlation scope keys: a NULL in any element
+ /// of a multi-column value key can make the tuple comparison UNKNOWN. For a
/// null-equal join, every nullable key needs the escape.
fn preserve_probe_nulls(
&self,
@@ -821,20 +822,23 @@ impl SharedBuildAccumulator {
// probe NULL makes `NOT IN` unknown for every build row. A null-equal join needs probe
// NULLs only to match an actual build-side NULL, so a NULL-free build keeps the filter
// at full selectivity.
- let needs_probe_nulls = self.null_aware
+ let needs_probe_nulls = self.null_aware.is_some()
|| (self.null_equality == NullEquality::NullEqualsNull
&& build_keys_have_null);
if !needs_probe_nulls {
return Ok(filter_expr);
}
- let keys = if self.null_aware {
- assert_or_internal_err!(
- !self.on_right.is_empty(),
- "null-aware join must have at least one probe key"
- );
- &self.on_right[..1]
- } else {
- self.on_right.as_slice()
+ let keys = match self.null_aware {
+ Some(mode) => {
+ let value_keys = mode.value_keys();
+ assert_or_internal_err!(
+ (1..=self.on_right.len()).contains(&value_keys),
+ "null-aware join must have between 1 and {} value keys, got {value_keys}",
+ self.on_right.len()
+ );
+ &self.on_right[..value_keys]
+ }
+ None => self.on_right.as_slice(),
};
// Only a key that can actually be NULL needs the disjunct; a NOT NULL key
@@ -894,7 +898,7 @@ pub(super) fn make_partitioned_accumulator_for_test(
probe_schema,
probe_range_partitioning: None,
null_equality: NullEquality::NullEqualsNothing,
- null_aware: false,
+ null_aware: None,
}
}
@@ -959,7 +963,7 @@ mod tests {
probe_schema: test_probe_schema(),
probe_range_partitioning: None,
null_equality: NullEquality::NullEqualsNothing,
- null_aware: false,
+ null_aware: None,
}
}
@@ -1447,7 +1451,10 @@ mod tests {
probe_schema,
probe_range_partitioning: None,
null_equality,
- null_aware,
+ null_aware: null_aware.then_some(NullAwareMode::LeftAnti {
+ correlated: false,
+ value_keys: 1,
+ }),
}
}
@@ -1624,61 +1631,90 @@ mod tests {
assert_eq!(format!("{widened}").matches("IS NULL").count(), 1);
}
- // Correlated null-aware LeftMark joins have multiple probe keys
- // (`on_right[0]` = scalar NOT IN value key, `on_right[1..]` = correlation
- // scope keys). The NULL escape must accept that shape and wrap only the
- // value key.
+ // A null-aware join with several probe keys has its `NOT IN` value keys
+ // first (`on_right[..V]`: one for a scalar `NOT IN`, one per tuple element
+ // for a multi-column one) and correlation scope keys after them. A NULL in
+ // any value key can make `NOT IN` UNKNOWN, so the NULL escape must wrap
+ // every value key and no scope key.
#[test]
- fn null_aware_multi_key_filter_escapes_value_key_only() {
- let on_right: Vec<PhysicalExprRef> = vec![
- Arc::new(Column::new("value_key", 0)),
- Arc::new(Column::new("scope_key", 1)),
- ];
- let dynamic_filter = test_dynamic_filter(&on_right);
- let acc = SharedBuildAccumulator {
- inner: Mutex::new(AccumulatorState {
- data: AccumulatedBuildData::CollectLeft {
- data: PartitionStatus::Pending,
- reported_count: 0,
- expected_reports: 1,
- },
- completion: CompletionState::Pending,
- }),
- completion_notify: Notify::new(),
- dynamic_filter,
- on_right,
- repartition_random_state: SeededRandomState::with_seed(1),
- probe_schema: Arc::new(Schema::new(vec![
- Field::new("value_key", DataType::Int32, true),
- // Keep the scope key nullable so this test proves it is not widened.
- Field::new("scope_key", DataType::Int32, true),
- ])),
- probe_range_partitioning: None,
- null_equality: NullEquality::NullEqualsNothing,
- null_aware: true,
- };
+ fn null_aware_multi_key_filter_escapes_value_keys_only() {
+ for value_keys in [1, 2] {
+ let mut on_right: Vec<PhysicalExprRef> = (0..value_keys)
+ .map(|i| Arc::new(Column::new(&format!("value_key_{i}"), i)) as _)
+ .collect();
+ on_right.push(Arc::new(Column::new("scope_key", value_keys)));
+ let mut fields: Vec<Field> = (0..value_keys)
+ .map(|i| Field::new(format!("value_key_{i}"), DataType::Int32, true))
+ .collect();
+ // Keep the scope key nullable so this test proves it is not widened.
+ fields.push(Field::new("scope_key", DataType::Int32, true));
+
+ let dynamic_filter = test_dynamic_filter(&on_right);
+ let acc = SharedBuildAccumulator {
+ inner: Mutex::new(AccumulatorState {
+ data: AccumulatedBuildData::CollectLeft {
+ data: PartitionStatus::Pending,
+ reported_count: 0,
+ expected_reports: 1,
+ },
+ completion: CompletionState::Pending,
+ }),
+ completion_notify: Notify::new(),
+ dynamic_filter,
+ on_right,
+ repartition_random_state: SeededRandomState::with_seed(1),
+ probe_schema: Arc::new(Schema::new(fields)),
+ probe_range_partitioning: None,
+ null_equality: NullEquality::NullEqualsNothing,
+ null_aware: Some(NullAwareMode::LeftMark {
+ correlated: true,
+ value_keys,
+ }),
+ };
- let two_key_bounds = PartitionBounds::new(vec![
- ColumnBounds::new(ScalarValue::Int32(Some(1)), ScalarValue::Int32(Some(5))),
- ColumnBounds::new(ScalarValue::Int32(Some(10)), ScalarValue::Int32(Some(20))),
- ]);
- acc.build_filter(FinalizeInput::CollectLeft(reported(
- PushdownStrategy::Empty,
- two_key_bounds,
- )))
- .unwrap();
+ let bounds = PartitionBounds::new(
+ (0..=value_keys as i32)
+ .map(|i| {
+ ColumnBounds::new(
+ ScalarValue::Int32(Some(10 * i)),
+ ScalarValue::Int32(Some(10 * i + 5)),
+ )
+ })
+ .collect(),
+ );
+ acc.build_filter(FinalizeInput::CollectLeft(reported(
+ PushdownStrategy::Empty,
+ bounds,
+ )))
+ .unwrap();
- let expr = current_expr(&acc);
- let or = binary_expr(&expr);
- assert_eq!(or.op(), &Operator::Or);
- let is_null = or
- .left()
- .downcast_ref::<IsNullExpr>()
- .expect("expected IS NULL escape as the left disjunct");
- let column = is_null
- .arg()
- .downcast_ref::<Column>()
- .expect("expected column under IS NULL");
- assert_eq!(column.index(), 0, "escape must target the NOT IN value key");
+ let expr = current_expr(&acc);
+ let or = binary_expr(&expr);
+ assert_eq!(or.op(), &Operator::Or);
+ let mut escaped = vec![];
+ collect_is_null_columns(or.left(), &mut escaped);
+ assert_eq!(
+ escaped,
+ (0..value_keys).collect::<Vec<_>>(),
+ "escape must target every NOT IN value key and nothing else"
+ );
+ }
+ }
+
+ /// Column indices under the `IS NULL` disjuncts of an `OR` chain.
+ fn collect_is_null_columns(expr: &Arc<dyn PhysicalExpr>, out: &mut Vec<usize>) {
+ if let Some(or) = expr.downcast_ref::<BinaryExpr>() {
+ assert_eq!(or.op(), &Operator::Or);
+ collect_is_null_columns(or.left(), out);
+ collect_is_null_columns(or.right(), out);
+ } else {
+ let column = expr
+ .downcast_ref::<IsNullExpr>()
+ .expect("expected IS NULL escape")
+ .arg()
+ .downcast_ref::<Column>()
+ .expect("expected column under IS NULL");
+ out.push(column.index());
+ }
}
}
diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs
index dc6a32fa14..fb2263ebcc 100644
--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs
@@ -27,7 +27,7 @@ use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};
use crate::joins::Map;
use crate::joins::MapOffset;
use crate::joins::PartitionMode;
-use crate::joins::hash_join::exec::{JoinLeftData, NullAwareMode};
+use crate::joins::hash_join::exec::{JoinLeftData, NullAwareMode, null_value_key_mask};
use crate::joins::hash_join::probe_completion::ProbeSideSummary;
use crate::joins::hash_join::shared_bounds::{
PartitionBounds, PartitionBuildData, SharedBuildAccumulator,
@@ -47,14 +47,17 @@ use crate::{
},
};
-use arrow::array::{Array, ArrayRef, UInt32Array, UInt64Array};
+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, UInt32Array, UInt64Array};
use arrow::buffer::{BooleanBuffer, NullBuffer};
+use arrow::compute::{filter, take};
use arrow::datatypes::{Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::{
JoinSide, JoinType, NullEquality, Result, internal_datafusion_err, internal_err,
};
+use datafusion_expr::{ColumnarValue, Operator};
use datafusion_physical_expr::PhysicalExprRef;
+use datafusion_physical_expr_common::datum::apply_cmp;
use datafusion_common::hash_utils::RandomState;
use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;
@@ -863,6 +866,7 @@ impl HashJoinStream {
mark_null_candidates_for_probe_batch(
build_side,
state,
+ mode.value_keys(),
self.filter.as_ref(),
self.join_type,
&self.random_state,
@@ -1149,7 +1153,7 @@ impl HashJoinStream {
// Null-aware joins post-process the build rows under SQL three-valued
// logic; see the helpers for the rules.
let (left_side, right_side, mark_column) = match self.null_aware {
- Some(NullAwareMode::LeftAnti { correlated }) => {
+ Some(NullAwareMode::LeftAnti { correlated, .. }) => {
let (left_side, right_side) = null_aware_left_anti_final_indices(
&build_side.left_data,
correlated,
@@ -1159,7 +1163,7 @@ impl HashJoinStream {
);
(left_side, right_side, None)
}
- Some(NullAwareMode::LeftMark { correlated }) => {
+ Some(NullAwareMode::LeftMark { correlated, .. }) => {
let mark_column = null_aware_left_mark_column(
&build_side.left_data,
correlated,
@@ -1273,10 +1277,18 @@ fn null_aware_skip_probe_batch(
NullAwareMode::RightAnti => left_data.build_side_has_null,
// Correlated joins decide UNKNOWN per build row instead, in
// `mark_null_candidates_for_probe_batch`.
- NullAwareMode::LeftAnti { correlated: true }
- | NullAwareMode::LeftMark { correlated: true } => false,
- NullAwareMode::LeftAnti { correlated: false }
- | NullAwareMode::LeftMark { correlated: false } => {
+ NullAwareMode::LeftAnti {
+ correlated: true, ..
+ }
+ | NullAwareMode::LeftMark {
+ correlated: true, ..
+ } => false,
+ NullAwareMode::LeftAnti {
+ correlated: false, ..
+ }
+ | NullAwareMode::LeftMark {
+ correlated: false, ..
+ } => {
// `on[0]` is the `NOT IN` value key for both modes.
let probe_key_column = &state.values[0];
let is_anti = matches!(mode, NullAwareMode::LeftAnti { .. });
@@ -1392,19 +1404,24 @@ fn null_aware_left_mark_column(
/// Records which build rows of a correlated null-aware join are UNKNOWN
/// candidates for this probe batch.
///
-/// Key layout: `on[0]` is the `NOT IN` value key, `on[1..]` the (possibly
-/// empty) correlation scope keys (see `HashJoinExec::null_aware`). An
-/// unmatched build row's `NOT IN` is UNKNOWN instead of TRUE (its mark is NULL
-/// instead of FALSE) when either:
-/// 1. its value key is NULL and any probe row in its correlation scope passes
-/// the join filter, or
-/// 2. some probe row in its correlation scope with a NULL value key passes the
-/// join filter.
+/// Key layout: `on[..V]` are the `NOT IN` value keys, `on[V..]` the (possibly
+/// empty) correlation scope keys (see `HashJoinExec::null_aware`). A row is
+/// NULL-valued when any of its value keys is NULL. An unmatched build row's
+/// `NOT IN` is UNKNOWN instead of TRUE (its mark is NULL instead of FALSE)
+/// when either:
+/// 1. it is NULL-valued and a probe row in its correlation scope passes the
+/// join filter, or
+/// 2. a NULL-valued probe row in its correlation scope passes the join filter,
+///
+/// and, for a multi-column value key, the two rows' value tuples are not a
+/// definite mismatch: every element pair is equal or involves a NULL. With a
+/// single value key a NULL on either side already rules out a mismatch.
///
/// Case 1 pairs the NULL-valued build rows with all probe rows; case 2 pairs
/// all build rows with the NULL-valued probe rows. Scope keys narrow these
/// pairs through a hash lookup; without scope keys every pair is a candidate.
-/// The join filter, if any, then decides which candidates count.
+/// The value tuples and then the join filter, if any, decide which candidates
+/// count.
///
/// A build row stays UNKNOWN once it is marked, so candidates whose build row
/// is already marked are skipped, and the join filter is not evaluated for
@@ -1414,6 +1431,7 @@ fn null_aware_left_mark_column(
fn mark_null_candidates_for_probe_batch(
build_side: &BuildSideReadyState,
state: &ProcessProbeBatchState,
+ num_value_keys: usize,
filter: Option<&JoinFilter>,
join_type: JoinType,
random_state: &RandomState,
@@ -1424,9 +1442,8 @@ fn mark_null_candidates_for_probe_batch(
) -> Result<()> {
let left_data = &build_side.left_data;
let null_value_build_rows = left_data.null_value_build_rows();
- let probe_value_key = &state.values[0];
- let probe_has_null_values = probe_value_key.logical_null_count() > 0;
- if null_value_build_rows.is_none() && !probe_has_null_values {
+ let probe_null_value_mask = null_value_key_mask(&state.values[..num_value_keys]);
+ if null_value_build_rows.is_none() && probe_null_value_mask.is_none() {
return Ok(());
}
@@ -1435,14 +1452,25 @@ fn mark_null_candidates_for_probe_batch(
state.values.len(),
"build/probe key counts must match"
);
- let build_scope_values = &left_data.values()[1..];
- let probe_scope_values = &state.values[1..];
+ let (build_value_keys, build_scope_values) =
+ left_data.values().split_at(num_value_keys);
+ let (probe_value_keys, probe_scope_values) = state.values.split_at(num_value_keys);
- // Keeps the candidate pairs that pass the join filter and marks their
- // build rows as UNKNOWN.
+ // Keeps the candidate pairs whose value tuples are not a definite mismatch
+ // and that pass the join filter, and marks their build rows as UNKNOWN.
let mut mark = |build_indices: UInt64Array, probe_indices: UInt32Array| {
let (build_indices, probe_indices) =
retain_unmarked(left_data, build_indices, probe_indices);
+ let (build_indices, probe_indices) = if num_value_keys > 1 {
+ retain_value_mismatch_free(
+ build_value_keys,
+ probe_value_keys,
+ build_indices,
+ probe_indices,
+ )?
+ } else {
+ (build_indices, probe_indices)
+ };
if build_indices.is_empty() {
return Ok(());
}
@@ -1471,8 +1499,8 @@ fn mark_null_candidates_for_probe_batch(
Ok(())
};
- // Case 1: build rows with a NULL value key are UNKNOWN as soon as any
- // probe row in their correlation scope passes the filter.
+ // Case 1: NULL-valued build rows are UNKNOWN as soon as a probe row in
+ // their correlation scope passes the checks in `mark`.
if let Some(null_rows) = null_value_build_rows {
match &null_rows.scope_map {
Some(scope_map) => {
@@ -1514,9 +1542,8 @@ fn mark_null_candidates_for_probe_batch(
}
// Case 2: NULL-valued probe rows make every build row in their correlation
- // scope that passes the filter an UNKNOWN candidate.
- if probe_has_null_values {
- let null_mask = arrow::compute::is_null(probe_value_key.as_ref())?;
+ // scope that passes the checks in `mark` UNKNOWN.
+ if let Some(null_mask) = probe_null_value_mask {
let null_probe_rows = UInt32Array::from_iter_values(
null_mask.values().set_indices().map(|i| i as u32),
);
@@ -1570,6 +1597,58 @@ fn mark_null_candidates_for_probe_batch(
Ok(())
}
+/// Keeps the candidate pairs whose multi-column `NOT IN` value tuples are not a
+/// definite mismatch, i.e. every element pair is equal or involves a NULL.
+///
+/// Such a pair compares UNKNOWN when some element is NULL (TRUE pairs, with no
+/// NULL, are found by the hash lookup instead), whereas a pair with a definite
+/// mismatch compares FALSE whatever its NULLs are: `(NULL, 1) = (2, 3)` is
+/// `UNKNOWN AND FALSE`, which is FALSE.
+///
+/// Elements are compared with SQL `=` (see [`apply_cmp`]), which, like the
+/// join's own key equality, treats `-0.0` and `+0.0` as equal.
+fn retain_value_mismatch_free(
+ build_value_keys: &[ArrayRef],
+ probe_value_keys: &[ArrayRef],
+ build_indices: UInt64Array,
+ probe_indices: UInt32Array,
+) -> Result<(UInt64Array, UInt32Array)> {
+ if build_indices.is_empty() {
+ return Ok((build_indices, probe_indices));
+ }
+ let mut keep: Option<BooleanBuffer> = None;
+ for (build, probe) in build_value_keys.iter().zip(probe_value_keys) {
+ let build = take(build.as_ref(), &build_indices, None)?;
+ let probe = take(probe.as_ref(), &probe_indices, None)?;
+ let eq = apply_cmp(
+ Operator::Eq,
+ &ColumnarValue::Array(build),
+ &ColumnarValue::Array(probe),
+ )?
+ .into_array(build_indices.len())?;
+ let eq = eq.as_boolean();
+ // A NULL comparison (an element involving NULL) is not a mismatch.
+ let not_mismatch = match eq.nulls() {
+ Some(nulls) => eq.values() | &!nulls.inner(),
+ None => eq.values().clone(),
+ };
+ keep = Some(match keep {
+ Some(keep) => &keep & &not_mismatch,
+ None => not_mismatch,
+ });
+ }
+ match keep {
+ Some(keep) if keep.count_set_bits() < keep.len() => {
+ let keep = BooleanArray::new(keep, None);
+ Ok((
+ filter(&build_indices, &keep)?.as_primitive().clone(),
+ filter(&probe_indices, &keep)?.as_primitive().clone(),
+ ))
+ }
+ _ => Ok((build_indices, probe_indices)),
+ }
+}
+
/// Calls `f` with all correlation-scope matches between `build_scope_values`
/// and `probe_scope_values`, as chunks of at most `batch_size` pairs of
/// (position in `build_scope_values`, position in `probe_scope_values`).
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index 38abf7c203..2e6eaea107 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -287,6 +287,10 @@ message JoinNode {
datafusion_common.NullEquality null_equality = 7;
LogicalExprNode filter = 8;
bool null_aware = 9;
+ // Number of `NOT IN` value keys of a null-aware join (see
+ // `Join::null_aware_value_keys`). Messages predating this field decode it
+ // as 0, which is read as 1: they can only hold scalar `NOT IN` joins.
+ uint32 null_aware_value_keys = 10;
}
enum AsOfMatchOperator {
@@ -1431,6 +1435,11 @@ message HashJoinExecNode {
// turning old plans into empty results. With `optional`, absent decodes to
// `None`, which is the correct reading of an older message.
optional uint64 fetch = 12;
+ // Number of `NOT IN` value keys of a null-aware join (see
+ // `HashJoinExec::null_aware_value_keys`). Messages predating this field
+ // decode it as 0, which is read as 1: they can only hold scalar `NOT IN`
+ // joins.
+ uint32 null_aware_value_keys = 13;
}
enum StreamPartitionMode {
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 4dfcc8a0d8..665d01eb96 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -9796,6 +9796,9 @@ impl serde::Serialize for HashJoinExecNode {
if self.fetch.is_some() {
len += 1;
}
+ if self.null_aware_value_keys != 0 {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.HashJoinExecNode", len)?;
if let Some(v) = self.left.as_ref() {
struct_ser.serialize_field("left", v)?;
@@ -9838,6 +9841,9 @@ impl serde::Serialize for HashJoinExecNode {
#[allow(clippy::needless_borrows_for_generic_args)]
struct_ser.serialize_field("fetch", ToString::to_string(&v).as_str())?;
}
+ if self.null_aware_value_keys != 0 {
+ struct_ser.serialize_field("nullAwareValueKeys", &self.null_aware_value_keys)?;
+ }
struct_ser.end()
}
}
@@ -9864,6 +9870,8 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {
"dynamic_filter",
"dynamicFilter",
"fetch",
+ "null_aware_value_keys",
+ "nullAwareValueKeys",
];
#[allow(clippy::enum_variant_names)]
@@ -9879,6 +9887,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {
NullAware,
DynamicFilter,
Fetch,
+ NullAwareValueKeys,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -9911,6 +9920,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {
"nullAware" | "null_aware" => Ok(GeneratedField::NullAware),
"dynamicFilter" | "dynamic_filter" => Ok(GeneratedField::DynamicFilter),
"fetch" => Ok(GeneratedField::Fetch),
+ "nullAwareValueKeys" | "null_aware_value_keys" => Ok(GeneratedField::NullAwareValueKeys),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -9941,6 +9951,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {
let mut null_aware__ = None;
let mut dynamic_filter__ = None;
let mut fetch__ = None;
+ let mut null_aware_value_keys__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Left => {
@@ -10014,6 +10025,14 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {
map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0)
;
}
+ GeneratedField::NullAwareValueKeys => {
+ if null_aware_value_keys__.is_some() {
+ return Err(serde::de::Error::duplicate_field("nullAwareValueKeys"));
+ }
+ null_aware_value_keys__ =
+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)
+ ;
+ }
}
}
Ok(HashJoinExecNode {
@@ -10028,6 +10047,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {
null_aware: null_aware__.unwrap_or_default(),
dynamic_filter: dynamic_filter__,
fetch: fetch__,
+ null_aware_value_keys: null_aware_value_keys__.unwrap_or_default(),
})
}
}
@@ -11601,6 +11621,9 @@ impl serde::Serialize for JoinNode {
if self.null_aware {
len += 1;
}
+ if self.null_aware_value_keys != 0 {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.JoinNode", len)?;
if let Some(v) = self.left.as_ref() {
struct_ser.serialize_field("left", v)?;
@@ -11635,6 +11658,9 @@ impl serde::Serialize for JoinNode {
if self.null_aware {
struct_ser.serialize_field("nullAware", &self.null_aware)?;
}
+ if self.null_aware_value_keys != 0 {
+ struct_ser.serialize_field("nullAwareValueKeys", &self.null_aware_value_keys)?;
+ }
struct_ser.end()
}
}
@@ -11660,6 +11686,8 @@ impl<'de> serde::Deserialize<'de> for JoinNode {
"filter",
"null_aware",
"nullAware",
+ "null_aware_value_keys",
+ "nullAwareValueKeys",
];
#[allow(clippy::enum_variant_names)]
@@ -11673,6 +11701,7 @@ impl<'de> serde::Deserialize<'de> for JoinNode {
NullEquality,
Filter,
NullAware,
+ NullAwareValueKeys,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -11703,6 +11732,7 @@ impl<'de> serde::Deserialize<'de> for JoinNode {
"nullEquality" | "null_equality" => Ok(GeneratedField::NullEquality),
"filter" => Ok(GeneratedField::Filter),
"nullAware" | "null_aware" => Ok(GeneratedField::NullAware),
+ "nullAwareValueKeys" | "null_aware_value_keys" => Ok(GeneratedField::NullAwareValueKeys),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -11731,6 +11761,7 @@ impl<'de> serde::Deserialize<'de> for JoinNode {
let mut null_equality__ = None;
let mut filter__ = None;
let mut null_aware__ = None;
+ let mut null_aware_value_keys__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Left => {
@@ -11787,6 +11818,14 @@ impl<'de> serde::Deserialize<'de> for JoinNode {
}
null_aware__ = Some(map_.next_value()?);
}
+ GeneratedField::NullAwareValueKeys => {
+ if null_aware_value_keys__.is_some() {
+ return Err(serde::de::Error::duplicate_field("nullAwareValueKeys"));
+ }
+ null_aware_value_keys__ =
+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)
+ ;
+ }
}
}
Ok(JoinNode {
@@ -11799,6 +11838,7 @@ impl<'de> serde::Deserialize<'de> for JoinNode {
null_equality: null_equality__.unwrap_or_default(),
filter: filter__,
null_aware: null_aware__.unwrap_or_default(),
+ null_aware_value_keys: null_aware_value_keys__.unwrap_or_default(),
})
}
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index f30a5d640b..5275c34268 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -439,6 +439,11 @@ pub struct JoinNode {
pub filter: ::core::option::Option<::prost::alloc::boxed::Box<LogicalExprNode>>,
#[prost(bool, tag = "9")]
pub null_aware: bool,
+ /// Number of `NOT IN` value keys of a null-aware join (see
+ /// `Join::null_aware_value_keys`). Messages predating this field decode it
+ /// as 0, which is read as 1: they can only hold scalar `NOT IN` joins.
+ #[prost(uint32, tag = "10")]
+ pub null_aware_value_keys: u32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct AsOfJoinNode {
@@ -2173,6 +2178,12 @@ pub struct HashJoinExecNode {
/// `None`, which is the correct reading of an older message.
#[prost(uint64, optional, tag = "12")]
pub fetch: ::core::option::Option<u64>,
+ /// Number of `NOT IN` value keys of a null-aware join (see
+ /// `HashJoinExec::null_aware_value_keys`). Messages predating this field
+ /// decode it as 0, which is read as 1: they can only hold scalar `NOT IN`
+ /// joins.
+ #[prost(uint32, tag = "13")]
+ pub null_aware_value_keys: u32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SymmetricHashJoinExecNode {
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index a1890b05c6..2e1811f76b 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -1062,16 +1062,28 @@ impl AsLogicalPlan for LogicalPlanNode {
// them silently loses both fields. Both sides of the round
// trip should already have validated keys, so we don't need
// the builder's normalization / equijoin-pair checks.
- Ok(LogicalPlan::Join(Join::try_new(
- Arc::new(left),
- Arc::new(right),
- on,
- filter,
- datafusion_expr::JoinType::from(join_type),
- JoinConstraint::from(join_constraint),
- NullEquality::from(null_equality),
- join.null_aware,
- )?))
+ Ok(LogicalPlan::Join(
+ Join::try_new(
+ Arc::new(left),
+ Arc::new(right),
+ on,
+ filter,
+ datafusion_expr::JoinType::from(join_type),
+ JoinConstraint::from(join_constraint),
+ NullEquality::from(null_equality),
+ join.null_aware,
+ )?
+ // Plans encoded before the field existed decode it as 0; they
+ // could only hold scalar `NOT IN` joins.
+ .with_null_aware_value_keys(
+ usize_from_wire(
+ join.null_aware_value_keys,
+ "JoinNode",
+ "null_aware_value_keys",
+ )?
+ .max(1),
+ ),
+ ))
}
LogicalPlanType::AsOfJoin(join) => {
let left_keys =
@@ -1756,6 +1768,7 @@ impl AsLogicalPlan for LogicalPlanNode {
join_constraint,
null_equality,
null_aware,
+ null_aware_value_keys,
// Not encoded; recomputed by `Join::try_new` on decode.
schema: _,
}) => {
@@ -1799,6 +1812,14 @@ impl AsLogicalPlan for LogicalPlanNode {
null_equality: null_equality.into(),
filter,
null_aware: *null_aware,
+ null_aware_value_keys: u32::try_from(
+ *null_aware_value_keys,
+ )
+ .map_err(|_| {
+ internal_datafusion_err!(
+ "JoinNode: null_aware_value_keys {null_aware_value_keys} does not fit in u32"
+ )
+ })?,
},
))),
})
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index c19b92579e..4a3e666932 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -3854,42 +3854,63 @@ async fn roundtrip_empty_table_scan_with_projection() -> Result<()> {
}
// Regression test for https://github.com/apache/datafusion/issues/22065:
-// the decoder must preserve `null_aware = true` (NOT IN semantics)
-// across a to_proto -> from_proto round trip. `null_equality` is at
-// its default (`NullEqualsNothing`).
+// the decoder must preserve `null_aware = true` (NOT IN semantics), and the
+// `NOT IN` value key count of a multi-column `(a, b) NOT IN` (which also holds
+// a correlation key), across a to_proto -> from_proto round trip.
+// `null_equality` is at its default (`NullEqualsNothing`).
#[tokio::test]
async fn roundtrip_join_null_aware() -> Result<()> {
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
use datafusion_expr::JoinType;
let ctx = SessionContext::new();
- let sql = "
- SELECT id
- FROM (VALUES (1), (2), (3)) AS t1(id)
- WHERE id NOT IN (
- SELECT bad_id
- FROM (VALUES (CAST(1 AS INT)), (CAST(NULL AS INT))) AS excludes(bad_id)
- )
- ";
-
- let df = ctx.sql(sql).await?;
- let plan = ctx.state().optimize(df.logical_plan())?;
-
- let mut found_null_aware = false;
- plan.apply(|n| {
- if let LogicalPlan::Join(j) = n
- && j.join_type == JoinType::LeftAnti
- && j.null_aware
- {
- found_null_aware = true;
- }
- Ok(TreeNodeRecursion::Continue)
- })?;
- assert!(found_null_aware);
+ let cases = [
+ (
+ "
+ SELECT id
+ FROM (VALUES (1), (2), (3)) AS t1(id)
+ WHERE id NOT IN (
+ SELECT bad_id
+ FROM (VALUES (CAST(1 AS INT)), (CAST(NULL AS INT))) AS excludes(bad_id)
+ )
+ ",
+ (1, 1),
+ ),
+ (
+ "
+ SELECT k
+ FROM (VALUES (1, 1, 2), (2, 3, 4)) AS t1(k, a, b)
+ WHERE (a, b) NOT IN (
+ SELECT x, y
+ FROM (VALUES (1, CAST(NULL AS INT), 2)) AS t2(k, x, y)
+ WHERE t2.k = t1.k
+ )
+ ",
+ (3, 2),
+ ),
+ ];
- let bytes = logical_plan_to_bytes(&plan)?;
- let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
- assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
+ for (sql, expected_keys) in cases {
+ let df = ctx.sql(sql).await?;
+ let plan = ctx.state().optimize(df.logical_plan())?;
+
+ // (number of equi-join keys, number of `NOT IN` value keys)
+ let mut null_aware_keys = None;
+ plan.apply(|n| {
+ if let LogicalPlan::Join(j) = n
+ && j.join_type == JoinType::LeftAnti
+ && j.null_aware
+ {
+ null_aware_keys = Some((j.on.len(), j.null_aware_value_keys));
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ assert_eq!(null_aware_keys, Some(expected_keys));
+
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+ assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
+ }
Ok(())
}
diff --git a/datafusion/sql/src/expr/subquery.rs b/datafusion/sql/src/expr/subquery.rs
index fb0b1207d6..ea4e3fb342 100644
--- a/datafusion/sql/src/expr/subquery.rs
+++ b/datafusion/sql/src/expr/subquery.rs
@@ -70,12 +70,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
let outer_ref_columns = sub_plan.all_out_ref_exprs();
planner_context.pop_outer_query_schema();
- self.validate_single_column(
- &sub_plan,
- &spans,
- "Too many columns! The subquery should only return one column",
- "Select only one column in the subquery",
- )?;
+ // A tuple `(a, b) IN (SELECT x, y ...)` compares one subquery column per
+ // tuple element (the counts are checked once the tuple is planned, see
+ // `in_subquery_tuple_values`); anything else compares a single value,
+ // so the subquery must return exactly one column.
+ if !matches!(expr, SQLExpr::Tuple(_)) {
+ self.validate_single_column(
+ &sub_plan,
+ &spans,
+ "Too many columns! The subquery should only return one column",
+ "Select only one column in the subquery",
+ )?;
+ }
let expr_obj = self.sql_to_expr(expr, input_schema, planner_context)?;
diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
index 2213f8bdab..916819fbb9 100644
--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
@@ -406,7 +406,87 @@ SELECT * FROM test_table t1 WHERE c1 NOT IN (SELECT c2 FROM test_table t2 WHERE
NULL 0
#############
-## Cleanup
+## Test 19: Multi-column NOT IN plans
+#############
+
+# Results under a batch size sweep are in null_aware_multi_column.slt. A tuple
+# NOT IN is a null-aware join whose leading keys are the tuple elements
+# (`value_keys`), followed by any correlation keys.
+
+statement ok
+CREATE TABLE multi_col_outer(k INT, a INT, b INT) AS VALUES
+(1, 1, 2),
+(2, NULL, 8);
+
+statement ok
+CREATE TABLE multi_col_inner(k INT, x INT, y INT) AS VALUES
+(1, 1, 2),
+(2, NULL, 4);
+
+query TT
+EXPLAIN SELECT * FROM multi_col_outer o
+WHERE (a, b) NOT IN (SELECT x, y FROM multi_col_inner i WHERE i.k = o.k);
+----
+logical_plan
+01)LeftAnti Join: o.a = __correlated_sq_1.x, o.b = __correlated_sq_1.y, o.k = __correlated_sq_1.k null_aware(value_keys=2)
+02)--SubqueryAlias: o
+03)----TableScan: multi_col_outer projection=[k, a, b]
+04)--SubqueryAlias: __correlated_sq_1
+05)----Projection: i.x, i.y, i.k
+06)------SubqueryAlias: i
+07)--------TableScan: multi_col_inner projection=[k, x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(a@1, x@0), (b@2, y@1), (k@0, k@2)], null_aware(value_keys=2)
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# A constant element is projected as an outer column so that it stays a join
+# key in its tuple position.
+query TT
+EXPLAIN SELECT * FROM multi_col_outer
+WHERE (a, 2) NOT IN (SELECT x, y FROM multi_col_inner);
+----
+logical_plan
+01)Projection: multi_col_outer.k, multi_col_outer.a, multi_col_outer.b
+02)--LeftAnti Join: multi_col_outer.a = __correlated_sq_1.x, __correlated_sq_1_value_1 = __correlated_sq_1.multi_col_inner.y null_aware(value_keys=2)
+03)----Projection: multi_col_outer.k, multi_col_outer.a, multi_col_outer.b, Int64(2) AS __correlated_sq_1_value_1
+04)------TableScan: multi_col_outer projection=[k, a, b]
+05)----SubqueryAlias: __correlated_sq_1
+06)------Projection: multi_col_inner.x, CAST(multi_col_inner.y AS Int64)
+07)--------TableScan: multi_col_inner projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(a@1, x@0), (__correlated_sq_1_value_1@3, multi_col_inner.y@1)], projection=[k@0, a@1, b@2], null_aware(value_keys=2)
+02)--ProjectionExec: expr=[k@0 as k, a@1 as a, b@2 as b, 2 as __correlated_sq_1_value_1]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)--ProjectionExec: expr=[x@0 as x, CAST(y@1 AS Int64) as multi_col_inner.y]
+05)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+# As a nullable value, the tuple NOT IN is a null-aware mark join.
+query TT
+EXPLAIN SELECT a, b, (a, b) NOT IN (SELECT x, y FROM multi_col_inner) AS r
+FROM multi_col_outer;
+----
+logical_plan
+01)Projection: multi_col_outer.a, multi_col_outer.b, NOT __correlated_sq_1.mark AS r
+02)--LeftMark Join: multi_col_outer.a = __correlated_sq_1.x, multi_col_outer.b = __correlated_sq_1.y null_aware(value_keys=2)
+03)----TableScan: multi_col_outer projection=[a, b]
+04)----SubqueryAlias: __correlated_sq_1
+05)------TableScan: multi_col_inner projection=[x, y]
+physical_plan
+01)ProjectionExec: expr=[a@0 as a, b@1 as b, NOT mark@2 as r]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(a@0, x@0), (b@1, y@1)], null_aware(value_keys=2)
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+DROP TABLE multi_col_outer;
+
+statement ok
+DROP TABLE multi_col_inner;
+
+#############
+## Cleanup (original tables)
#############
statement ok
diff --git a/datafusion/sqllogictest/test_files/null_aware_multi_column.slt b/datafusion/sqllogictest/test_files/null_aware_multi_column.slt
new file mode 100644
index 0000000000..917d1caec9
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/null_aware_multi_column.slt
@@ -0,0 +1,595 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Multi-column `(a, b) NOT IN (SELECT x, y ...)` under SQL three-valued logic.
+# A tuple comparison is UNKNOWN only when no element pair is a definite
+# mismatch and at least one of them involves a NULL: `(NULL, 8) = (1, 2)` is
+# `UNKNOWN AND FALSE`, which is FALSE, while `(NULL, 2) = (1, 2)` is UNKNOWN.
+# The batch size sweep exercises the chunked UNKNOWN candidate pairing of the
+# null-aware hash join. Plans are checked in null_aware_anti_join.slt.
+# Matrix rules: no EXPLAIN, no in-file SET of a swept knob, rowsort every
+# multi-row query.
+
+# configMatrix: datafusion.execution.batch_size=1,2,100,8192
+
+statement ok
+CREATE TABLE mc_outer(a INT, b INT, value TEXT) AS VALUES
+(1, 2, 'x'),
+(3, 4, 'y'),
+(5, 6, 'z'),
+(NULL, 8, 'w'),
+(3, NULL, 'v'),
+(NULL, NULL, 'u');
+
+statement ok
+CREATE TABLE mc_inner_null(x INT, y INT) AS VALUES
+(1, 2),
+(NULL, 4);
+
+statement ok
+CREATE TABLE mc_inner_no_null(x INT, y INT) AS VALUES
+(1, 2);
+
+statement ok
+CREATE TABLE mc_empty(x INT, y INT);
+
+#############
+## NULL in the subquery
+#############
+
+# (1, 2) matches. (3, 4) vs (NULL, 4) is `UNKNOWN AND TRUE`. (3, NULL) vs
+# (NULL, 4) and (NULL, NULL) vs (1, 2) have no definite mismatch.
+# (5, 6) and (NULL, 8) mismatch both inner rows.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_inner_null);
+----
+5 6 z
+NULL 8 w
+
+#############
+## NULL only in the outer tuple
+#############
+
+# (3, NULL) and (NULL, 8) have a definite mismatch with (1, 2), so they are
+# kept; (NULL, NULL) vs (1, 2) is UNKNOWN.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_inner_no_null);
+----
+3 4 y
+3 NULL v
+5 6 z
+NULL 8 w
+
+# `NOT IN (<empty>)` is TRUE for every row, NULLs included.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_empty);
+----
+1 2 x
+3 4 y
+3 NULL v
+5 6 z
+NULL 8 w
+NULL NULL u
+
+#############
+## Three columns
+#############
+
+statement ok
+CREATE TABLE mc3_outer(a INT, b INT, c INT) AS VALUES
+(1, 2, 3),
+(4, 5, 6),
+(7, 8, 9);
+
+statement ok
+CREATE TABLE mc3_inner(x INT, y INT, z INT) AS VALUES
+(1, 2, 3),
+(4, NULL, 6);
+
+# (1, 2, 3) matches; (4, 5, 6) vs (4, NULL, 6) is UNKNOWN.
+query III rowsort
+SELECT * FROM mc3_outer
+WHERE (a, b, c) NOT IN (SELECT x, y, z FROM mc3_inner);
+----
+7 8 9
+
+#############
+## Constant elements
+#############
+
+# The constant is a join key in its tuple position. (1, 2) matches;
+# (NULL, 2) vs (1, 2) is `UNKNOWN AND TRUE`; (3, 2) and (5, 2) mismatch both
+# inner rows.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, 2) NOT IN (SELECT x, y FROM mc_inner_null);
+----
+3 4 y
+3 NULL v
+5 6 z
+
+# An all-literal tuple is folded into a struct literal before decorrelation.
+# (5, 6) mismatches both inner rows, so every outer row is kept.
+query T rowsort
+SELECT value FROM mc_outer
+WHERE (5, 6) NOT IN (SELECT x, y FROM mc_inner_null);
+----
+u
+v
+w
+x
+y
+z
+
+query BBB
+SELECT
+ (1, 2) NOT IN (SELECT x, y FROM mc_inner_null) AS matched,
+ (3, 4) NOT IN (SELECT x, y FROM mc_inner_null) AS unknown,
+ (5, 6) NOT IN (SELECT x, y FROM mc_inner_null) AS mismatched;
+----
+false NULL true
+
+#############
+## Correlated subqueries
+#############
+
+statement ok
+CREATE TABLE mc_corr_outer(a INT, b INT, grp INT) AS VALUES
+(1, 2, 1),
+(1, 3, 1),
+(2, 5, 1),
+(NULL, 9, 2),
+(NULL, 8, 2),
+(4, 9, 3),
+(4, 9, NULL);
+
+statement ok
+CREATE TABLE mc_corr_inner(x INT, y INT, grp INT) AS VALUES
+(1, 2, 1),
+(NULL, 3, 1),
+(4, 9, 2);
+
+# grp 1 holds (1, 2) and (NULL, 3); grp 2 holds (4, 9).
+# (1, 2, 1) matches; (1, 3, 1) vs (NULL, 3) and (NULL, 9, 2) vs (4, 9) are
+# UNKNOWN; (2, 5, 1) and (NULL, 8, 2) mismatch every row in scope; grp 3 and a
+# NULL grp have an empty scope.
+query III rowsort
+SELECT * FROM mc_corr_outer o
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_corr_inner i WHERE i.grp = o.grp);
+----
+2 5 1
+4 9 3
+4 9 NULL
+NULL 8 2
+
+query IIIB rowsort
+SELECT a, b, grp,
+ (a, b) NOT IN (SELECT x, y FROM mc_corr_inner i WHERE i.grp = o.grp) AS r
+FROM mc_corr_outer o;
+----
+1 2 1 false
+1 3 1 NULL
+2 5 1 true
+4 9 3 true
+4 9 NULL true
+NULL 8 2 true
+NULL 9 2 NULL
+
+statement ok
+CREATE TABLE mc_lt_outer(k INT, a INT, b INT) AS VALUES
+(1, 1, 2),
+(2, 1, 2),
+(3, NULL, 5),
+(4, 7, 7),
+(5, 4, 2);
+
+statement ok
+CREATE TABLE mc_lt_inner(k INT, x INT, y INT) AS VALUES
+(1, 1, 2),
+(3, NULL, 2),
+(3, 9, 5);
+
+# A non-equality correlation. k=1 has an empty scope; k=2 matches (1, 2);
+# (NULL, 5) and (7, 7) mismatch every row in scope; (4, 2) vs (NULL, 2) at
+# k=3 is `UNKNOWN AND TRUE`.
+query III rowsort
+SELECT * FROM mc_lt_outer o
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_lt_inner i WHERE i.k < o.k);
+----
+1 1 2
+3 NULL 5
+4 7 7
+
+#############
+## NOT IN as a nullable value
+#############
+
+# (1, 2) matches: false. (3, 4), (3, NULL) and (NULL, NULL) are UNKNOWN: NULL.
+# (5, 6) and (NULL, 8) mismatch both inner rows: true.
+query IIB rowsort
+SELECT a, b, (a, b) NOT IN (SELECT x, y FROM mc_inner_null) AS r
+FROM mc_outer;
+----
+1 2 false
+3 4 NULL
+3 NULL NULL
+5 6 true
+NULL 8 true
+NULL NULL NULL
+
+# Under OR only a TRUE `NOT IN` keeps a row on its own.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_inner_null) OR value = 'y';
+----
+3 4 y
+5 6 z
+NULL 8 w
+
+#############
+## Element types
+#############
+
+statement ok
+CREATE TABLE mc_wide_inner(x BIGINT, y DOUBLE) AS VALUES
+(1, 2.0),
+(3, NULL);
+
+# Each element is coerced to its column. (1, 2) matches (1, 2.0); (3, 4),
+# (3, NULL), (NULL, 8) and (NULL, NULL) have no definite mismatch with
+# (3, NULL) or (1, 2.0).
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_wide_inner);
+----
+5 6 z
+
+statement ok
+CREATE TABLE mc_float_outer(a DOUBLE, b DOUBLE) AS VALUES
+(NULL, 0.0),
+(1.0, 0.0),
+(2.0, 5.0);
+
+statement ok
+CREATE TABLE mc_float_inner(x DOUBLE, y DOUBLE) AS VALUES
+(1.0, arrow_cast('-0.0', 'Float64'));
+
+statement ok
+CREATE TABLE mc_float_inner_null(x DOUBLE, y DOUBLE) AS VALUES
+(NULL, arrow_cast('-0.0', 'Float64'));
+
+# `0.0 = -0.0`, so (1, 0) matches and (NULL, 0) vs (1, -0.0) is
+# `UNKNOWN AND TRUE`.
+query RR rowsort
+SELECT * FROM mc_float_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_float_inner);
+----
+2 5
+
+query RRB rowsort
+SELECT a, b, (a, b) NOT IN (SELECT x, y FROM mc_float_inner) FROM mc_float_outer;
+----
+1 0 false
+2 5 true
+NULL 0 NULL
+
+query RR rowsort
+SELECT * FROM mc_float_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_float_inner_null);
+----
+2 5
+
+# A null-aware hash join reads its first keys as the tuple elements. A
+# RunEndEncoded element is not hashable, so it would stay in the join filter
+# and a correlation key would take its key position; the tuple is rejected
+# instead.
+statement ok
+CREATE TABLE mc_ree_outer AS
+SELECT k, arrow_cast(a, 'RunEndEncoded("run_ends": non-null Int32, "values": Int32)') AS a, b
+FROM mc_lt_outer;
+
+statement ok
+CREATE TABLE mc_ree_inner AS
+SELECT k, arrow_cast(x, 'RunEndEncoded("run_ends": non-null Int32, "values": Int32)') AS x, y
+FROM mc_lt_inner;
+
+statement error Multi-column NOT IN subquery is not supported when a tuple element is not a hashable equi-join key
+SELECT * FROM mc_ree_outer o
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_ree_inner i WHERE i.k = o.k);
+
+# A scalar NOT IN has a fallback for this case. k=1 holds 1; k=3 holds NULL
+# and 9, so its NULL value is UNKNOWN; the other rows have an empty scope.
+query I?I rowsort
+SELECT * FROM mc_ree_outer o
+WHERE a NOT IN (SELECT x FROM mc_ree_inner i WHERE i.k = o.k);
+----
+2 1 2
+4 7 7
+5 4 2
+
+#############
+## Nullability of a tuple IN
+#############
+
+# The tuple `IN` is NULL whenever an element or a subquery column is NULL and
+# no element pair is a definite mismatch, so `IS [NOT] NULL` over it must not
+# be folded away even though both subquery columns here are NOT NULL.
+# (NULL, 8) vs (5, 8) and (NULL, NULL) vs (5, 8) are UNKNOWN; the other rows
+# have a definite mismatch.
+query TBB rowsort
+SELECT
+ value,
+ ((a, b) IN (SELECT 5, 8)) IS NULL AS unk,
+ ((a, b) NOT IN (SELECT 5, 8)) IS NOT NULL AS known
+FROM mc_outer;
+----
+u true false
+v false true
+w true false
+x false true
+y false true
+z false true
+
+# The examples in docs/source/user-guide/sql/subqueries.md. A tuple compared
+# with a subquery follows SQL row comparison, unlike the list form
+# `(1, NULL) IN ((1, NULL))`, which is true.
+query BB
+SELECT (1, NULL) IN (SELECT 1, NULL) AS in_null, (NULL, 1) NOT IN (SELECT 2, 3) AS not_in;
+----
+NULL true
+
+#############
+## Subqueries whose top is not a projection
+#############
+
+statement ok
+CREATE TABLE mc_top_inner(x INT, y INT, k INT) AS VALUES
+(1, 2, 1),
+(1, 2, 2),
+(NULL, 4, 1),
+(9, 9, 3);
+
+# DISTINCT: {(1, 2), (NULL, 4), (9, 9)}. (1, 2) matches; (3, 4), (3, NULL)
+# and (NULL, NULL) are UNKNOWN against (NULL, 4) or (1, 2).
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT DISTINCT x, y FROM mc_top_inner);
+----
+5 6 z
+NULL 8 w
+
+# GROUP BY k: {(1, 4), (1, 2), (9, 9)}, with no NULL. Only (1, 2) matches and
+# only (NULL, NULL) has no definite mismatch.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT max(x), max(y) FROM mc_top_inner GROUP BY k);
+----
+3 4 y
+3 NULL v
+5 6 z
+NULL 8 w
+
+# ORDER BY ... LIMIT: the two k = 1 rows, {(1, 2), (NULL, 4)}.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_top_inner ORDER BY k LIMIT 2);
+----
+5 6 z
+NULL 8 w
+
+#############
+## Utf8 elements
+#############
+
+statement ok
+CREATE TABLE mc_str_outer(a TEXT, b TEXT) AS VALUES
+(NULL, 'x'),
+('p', 'x'),
+('q', 'y'),
+('r', NULL),
+('s', 'z');
+
+statement ok
+CREATE TABLE mc_str_inner(x TEXT, y TEXT) AS VALUES
+('p', 'x'),
+(NULL, 'y');
+
+# ('p', 'x') matches; (NULL, 'x') vs ('p', 'x'), ('q', 'y') vs (NULL, 'y') and
+# ('r', NULL) vs (NULL, 'y') are UNKNOWN; ('s', 'z') mismatches both.
+query TT rowsort
+SELECT * FROM mc_str_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_str_inner);
+----
+s z
+
+#############
+## Mark join with a non-equality correlation
+#############
+
+# The join filter decides the scope of each row: k=1 has an empty scope, k=2
+# matches (1, 2), (NULL, 5) and (7, 7) mismatch every row in scope, and
+# (4, 2) vs (NULL, 2) at k=3 is UNKNOWN.
+query IB rowsort
+SELECT k, (a, b) NOT IN (SELECT x, y FROM mc_lt_inner i WHERE i.k < o.k) AS r
+FROM mc_lt_outer o;
+----
+1 true
+2 false
+3 true
+4 true
+5 NULL
+
+#############
+## Session time zone
+#############
+
+# A naive timestamp is compared with a time zone aware one in the session time
+# zone, for every tuple element as for a scalar `IN`. In +08 the naive
+# 2024-11-01 00:00 is not the aware 2024-11-01 00:00-04:00 (it would be if the
+# naive value were read in the aware value's own time zone).
+statement ok
+SET TIME ZONE = '+08'
+
+query II
+WITH timestamps(ts_tz, ts) AS (
+ VALUES (
+ arrow_cast('2024-11-01T00:00:00-04:00', 'Timestamp(Nanosecond, Some("America/New_York"))'),
+ '2024-11-01T00:00:00'::timestamp
+ )
+)
+SELECT
+ (SELECT count(*) FROM timestamps WHERE (ts_tz, 1) IN (SELECT ts, 1 FROM timestamps)) AS in_count,
+ (SELECT count(*) FROM timestamps WHERE (ts_tz, 1) NOT IN (SELECT ts, 1 FROM timestamps)) AS not_in_count;
+----
+0 1
+
+statement ok
+RESET datafusion.execution.time_zone
+
+#############
+## Tuple and subquery column counts
+#############
+
+statement error The number of columns in the tuple \(2\) must match the number of columns in the subquery \(3\)
+SELECT * FROM mc_outer WHERE (a, b) NOT IN (SELECT x, y, x AS z FROM mc_inner_no_null);
+
+statement error The number of columns in the tuple \(3\) must match the number of columns in the subquery \(2\)
+SELECT * FROM mc_outer WHERE (a, b, a) NOT IN (SELECT x, y FROM mc_inner_no_null);
+
+statement error Too many columns! The subquery should only return one column
+SELECT * FROM mc_outer WHERE a NOT IN (SELECT x, y FROM mc_inner_no_null);
+
+# Against a single struct column the tuple is one struct value, not a
+# multi-column `IN`.
+query IIT rowsort
+SELECT * FROM mc_outer
+WHERE (a, b) IN (SELECT struct(x, y) FROM mc_inner_no_null);
+----
+1 2 x
+
+#############
+## Placeholders take their tuple column's type
+#############
+
+statement ok
+PREPARE mc_tuple_not_in AS SELECT value FROM mc_outer
+WHERE ($1, b) NOT IN (SELECT x, y FROM mc_inner_no_null);
+
+# ($1 = 1): (1, 2) matches; (1, NULL) vs (1, 2) is UNKNOWN; the rest mismatch.
+query T rowsort
+EXECUTE mc_tuple_not_in(1);
+----
+w
+y
+z
+
+statement ok
+DEALLOCATE mc_tuple_not_in;
+
+statement ok
+CREATE TABLE mc_p1(id INT) AS VALUES (1), (3);
+
+statement ok
+CREATE TABLE mc_p2(id INT) AS VALUES (1), (3);
+
+# Two subquery columns named `id` under different qualifiers.
+statement ok
+PREPARE mc_tuple_qualified AS SELECT value FROM mc_outer
+WHERE ($1, b) NOT IN (SELECT p1.id, p2.id FROM mc_p1 p1 JOIN mc_p2 p2 ON p1.id = p2.id);
+
+# ($1 = 1) vs {(1, 1), (3, 3)}: a NULL b is UNKNOWN against (1, 1), every
+# other b mismatches both rows.
+query T rowsort
+EXECUTE mc_tuple_qualified(1);
+----
+w
+x
+y
+z
+
+statement ok
+DEALLOCATE mc_tuple_qualified;
+
+#############
+## Cleanup
+#############
+
+statement ok
+DROP TABLE mc_outer;
+
+statement ok
+DROP TABLE mc_inner_null;
+
+statement ok
+DROP TABLE mc_inner_no_null;
+
+statement ok
+DROP TABLE mc_empty;
+
+statement ok
+DROP TABLE mc3_outer;
+
+statement ok
+DROP TABLE mc3_inner;
+
+statement ok
+DROP TABLE mc_corr_outer;
+
+statement ok
+DROP TABLE mc_corr_inner;
+
+statement ok
+DROP TABLE mc_lt_outer;
+
+statement ok
+DROP TABLE mc_lt_inner;
+
+statement ok
+DROP TABLE mc_wide_inner;
+
+statement ok
+DROP TABLE mc_float_outer;
+
+statement ok
+DROP TABLE mc_float_inner;
+
+statement ok
+DROP TABLE mc_float_inner_null;
+
+statement ok
+DROP TABLE mc_ree_outer;
+
+statement ok
+DROP TABLE mc_ree_inner;
+
+statement ok
+DROP TABLE mc_top_inner;
+
+statement ok
+DROP TABLE mc_str_outer;
+
+statement ok
+DROP TABLE mc_str_inner;
+
+statement ok
+DROP TABLE mc_p1;
+
+statement ok
+DROP TABLE mc_p2;
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index c1fd856497..09f648df27 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -3014,6 +3014,202 @@ DROP TABLE gs_outer;
statement ok
DROP TABLE gs_inner;
+#############
+## Multi-column IN (LeftSemi) Tests
+#############
+## These tests verify that multi-column IN subqueries work correctly
+## Multi-column IN uses LeftSemi join (not null-aware)
+
+#############
+## Test 1: Basic two-column IN
+#############
+
+statement ok
+CREATE TABLE multi_in_left(a INT, b INT, value TEXT) AS VALUES
+(1, 2, 'match'),
+(3, 4, 'no_match'),
+(5, 6, 'match');
+
+statement ok
+CREATE TABLE multi_in_right(x INT, y INT) AS VALUES
+(1, 2),
+(5, 6);
+
+# Should return rows where (a, b) matches (x, y)
+query IIT rowsort
+SELECT * FROM multi_in_left
+WHERE (a, b) IN (SELECT x, y FROM multi_in_right);
+----
+1 2 match
+5 6 match
+
+#############
+## Test 2: Multi-column IN with no matches
+#############
+
+statement ok
+CREATE TABLE multi_in_right_no_match(x INT, y INT) AS VALUES
+(10, 20),
+(30, 40);
+
+# Should return empty result
+query IIT rowsort
+SELECT * FROM multi_in_left
+WHERE (a, b) IN (SELECT x, y FROM multi_in_right_no_match);
+----
+
+#############
+## Test 3: Multi-column IN with NULL values
+#############
+## Note: Unlike NOT IN, regular IN does NOT use null-aware semantics
+## NULL = NULL is always FALSE (not unknown) in regular semi joins
+
+statement ok
+CREATE TABLE multi_in_left_null(a INT, b INT, value TEXT) AS VALUES
+(1, 2, 'x'),
+(3, NULL, 'y'),
+(NULL, 6, 'z');
+
+statement ok
+CREATE TABLE multi_in_right_null(x INT, y INT) AS VALUES
+(1, 2),
+(NULL, 4);
+
+# Should return only (1, 2, 'x')
+# (3, NULL, 'y') doesn't match because NULL doesn't equal anything
+# (NULL, 6, 'z') doesn't match because NULL doesn't equal anything
+query IIT rowsort
+SELECT * FROM multi_in_left_null
+WHERE (a, b) IN (SELECT x, y FROM multi_in_right_null);
+----
+1 2 x
+
+#############
+## Test 4: Three-column IN
+#############
+
+statement ok
+CREATE TABLE three_col_left(a INT, b INT, c INT, value TEXT) AS VALUES
+(1, 2, 3, 'match1'),
+(4, 5, 6, 'no_match'),
+(7, 8, 9, 'match2');
+
+statement ok
+CREATE TABLE three_col_right(x INT, y INT, z INT) AS VALUES
+(1, 2, 3),
+(7, 8, 9);
+
+# Should return rows with matching three-column tuples
+query IIIT rowsort
+SELECT * FROM three_col_left
+WHERE (a, b, c) IN (SELECT x, y, z FROM three_col_right);
+----
+1 2 3 match1
+7 8 9 match2
+
+#############
+## Test 5: Correlated multi-column IN
+#############
+
+statement ok
+CREATE TABLE correlated_outer(id INT, a INT, b INT) AS VALUES
+(1, 10, 20),
+(2, 30, 40),
+(3, 10, 20);
+
+statement ok
+CREATE TABLE correlated_inner(id INT, x INT, y INT) AS VALUES
+(1, 10, 20),
+(1, 30, 40),
+(2, 50, 60),
+(3, 10, 20);
+
+# Should return outer rows where (a, b) matches (x, y) for the same id
+query III rowsort
+SELECT * FROM correlated_outer o
+WHERE (a, b) IN (
+ SELECT x, y FROM correlated_inner i WHERE i.id = o.id
+);
+----
+1 10 20
+3 10 20
+
+#############
+## Test 6: Verify logical plan shows LeftSemi join with multiple conditions
+#############
+
+query TT
+EXPLAIN SELECT * FROM multi_in_left
+WHERE (a, b) IN (SELECT x, y FROM multi_in_right);
+----
+logical_plan
+01)LeftSemi Join: multi_in_left.a = __correlated_sq_1.x, multi_in_left.b = __correlated_sq_1.y
+02)--TableScan: multi_in_left projection=[a, b, value]
+03)--SubqueryAlias: __correlated_sq_1
+04)----TableScan: multi_in_right projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(x@0, a@0), (y@1, b@1)]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+#############
+## Test 7: Multi-column IN with empty subquery
+#############
+
+statement ok
+CREATE TABLE multi_in_right_empty(x INT, y INT);
+
+# Should return empty result (empty subquery)
+query IIT rowsort
+SELECT * FROM multi_in_left
+WHERE (a, b) IN (SELECT x, y FROM multi_in_right_empty);
+----
+
+#############
+## Test 8: Multi-column IN with WHERE clause in subquery
+#############
+
+# The filter drops (1, 2) from the subquery, so only (5, 6) matches
+query IIT rowsort
+SELECT * FROM multi_in_left
+WHERE (a, b) IN (SELECT x, y FROM multi_in_right WHERE x > 1);
+----
+5 6 match
+
+#############
+## Cleanup
+#############
+
+statement ok
+DROP TABLE multi_in_left;
+
+statement ok
+DROP TABLE multi_in_right;
+
+statement ok
+DROP TABLE multi_in_right_no_match;
+
+statement ok
+DROP TABLE multi_in_left_null;
+
+statement ok
+DROP TABLE multi_in_right_null;
+
+statement ok
+DROP TABLE three_col_left;
+
+statement ok
+DROP TABLE three_col_right;
+
+statement ok
+DROP TABLE correlated_outer;
+
+statement ok
+DROP TABLE correlated_inner;
+
+statement ok
+DROP TABLE multi_in_right_empty;
+
# Regression test for #25507: a correlated filter below the side of an outer
# join that the join fills with NULLs must not be pulled above the join. The
# filter decides which rows of the other side are unmatched, so once it is
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index fceda8f611..11a63a5aa5 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -816,6 +816,38 @@ Wire compatibility is directional:
ignores the unknown field but cannot preserve the qualifier required by the
expressions, which can cause resolution failure or incorrect rebinding.
+### Null-aware joins record their `NOT IN` value key count
+
+Multi-column `(a, b) NOT IN (SELECT x, y ...)` is now planned as a null-aware
+join. Its first equi-join keys are the tuple elements, followed by any
+correlation keys, so a null-aware join now records how many of its leading keys
+are `NOT IN` value keys:
+
+- `datafusion_expr::logical_plan::Join` has a new `null_aware_value_keys: usize`
+ field.
+- `HashJoinExec` has a new `null_aware_value_keys: usize` field, set with
+ `HashJoinExecBuilder::with_null_aware_value_keys`.
+
+The count is only read when `null_aware` is set. `Join::try_new`, the
+`LogicalPlanBuilder` join methods and `HashJoinExec::try_new` set it to `1`,
+the scalar `NOT IN` of earlier releases. Code constructing `Join` with a struct
+literal must initialize the field:
+
+```rust,ignore
+let join = Join {
+ // ...
+ null_aware,
+ null_aware_value_keys: 1,
+};
+```
+
+The generated protobuf `JoinNode` and `HashJoinExecNode` types have a new
+`null_aware_value_keys` field, which struct literals must provide too. A 56.0
+reader decodes a payload without the field (value `0`) as a scalar `NOT IN`. A
+55.0 reader must not consume a 56.0 plan holding a multi-column `NOT IN`: it
+ignores the unknown field and would read the extra tuple elements as
+correlation keys, returning wrong results.
+
### `FunctionalDependence` has a new `null_equality` field
`FunctionalDependence` now records the NULL semantics its dependency holds under.
diff --git a/docs/source/user-guide/sql/subqueries.md b/docs/source/user-guide/sql/subqueries.md
index cba15f1bfd..f96d9a7045 100644
--- a/docs/source/user-guide/sql/subqueries.md
+++ b/docs/source/user-guide/sql/subqueries.md
@@ -78,8 +78,12 @@ the results of a subquery or list of values.
```sql
expression [NOT] IN (subquery|list-literal)
+(expression[, ...]) [NOT] IN (subquery)
```
+The tuple form compares each tuple element with the subquery column at the
+same position, so the subquery must return one column per element.
+
#### Examples
```sql
@@ -114,6 +118,20 @@ SELECT (1, NULL) IN ((1, NULL));
-- true
```
+A tuple compared with a subquery instead follows SQL row comparison: two rows
+are equal when every element pair is equal, and the comparison is UNKNOWN
+(`NULL`) when no element pair differs but at least one involves a `NULL`. So
+the same values compared with a subquery give `NULL`, and `NOT IN` keeps a row
+only when every subquery row differs from it in some non-`NULL` element:
+
+```sql
+SELECT (1, NULL) IN (SELECT 1, NULL);
+-- NULL
+
+SELECT (NULL, 1) NOT IN (SELECT 2, 3);
+-- true
+```
+
## SELECT clause subqueries
`SELECT` clause subqueries use values returned from the inner query as part
@@ -242,7 +260,8 @@ WHERE
#### `WHERE` clause with non-scalar subquery
Non-scalar subqueries must use the `[NOT] IN` or `[NOT] EXISTS` operators and
-can only return a single column.
+can only return a single column, or one column per element of a tuple
+`(expression[, ...]) [NOT] IN (subquery)`.
The values in the returned column are evaluated as a list.
The following query returns all rows with `column_2` values in table `x` that
@@ -357,7 +376,8 @@ HAVING
#### `HAVING` clause with a non-scalar subquery
Non-scalar subqueries must use the `[NOT] IN` or `[NOT] EXISTS` operators and
-can only return a single column.
+can only return a single column, or one column per element of a tuple
+`(expression[, ...]) [NOT] IN (subquery)`.
The values in the returned column are evaluated as a list.
The following query calculates the averages of even and odd numbers in table `y`