Skip to content

Apache DataFusion SQL Query Engine

perf: read NOT NULL tuple elements of a multi-column NOT IN as scope keys (#26214)

Liang-Chi Hsieh132234d8202d
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q11.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q11.benchmark
index 067616db49..cf4b5a30d2 100644
--- a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q11.benchmark
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q11.benchmark
@@ -25,9 +25,10 @@ expect_plan HashJoinExec
run
-- Q11: uncorrelated multi-column NOT IN with 1% NULL in one subquery element.
--- Each NULL-valued subquery row is paired with every outer row and the pair is
--- kept as UNKNOWN only when the other element matches, so the cost grows with
--- (outer rows) x (NULL-valued subquery rows). Compare with Q10 and Q12.
+-- `z` is NOT NULL on both sides, so the join reads it as a scope key: each
+-- NULL-valued subquery row is paired only with the outer rows with the same
+-- `z`, through a hash lookup, so the cost grows with (NULL-valued subquery
+-- rows) x (outer rows per `z` value). Compare with Q10 and Q12.
SELECT count(*)
FROM small_outer o
WHERE (o.id_n0, o.z) NOT IN (SELECT i.id_n1, i.z FROM small_inner i);
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark
index 8197a47698..80f6ed99c2 100644
--- a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark
@@ -26,10 +26,10 @@ expect_plan HashJoinExec
run
-- Q12: uncorrelated multi-column NOT IN with 1% NULL in one outer element,
--- the mirror of Q11. Each NULL-valued outer row is paired with every subquery
--- row and is UNKNOWN as soon as the other element of one of them matches: the
--- cost grows with (NULL-valued outer rows) x (subquery rows). Compare with Q10
--- and Q11.
+-- the mirror of Q11. `z` is NOT NULL on both sides, so the join reads it as a
+-- scope key: each NULL-valued outer row is paired only with the subquery rows
+-- with the same `z`, so the cost grows with (NULL-valued outer rows) x
+-- (subquery rows per `z` value). Compare with Q10 and Q11.
SELECT count(*)
FROM small_outer o
WHERE (o.id_n1, o.z) NOT IN (SELECT i.id_n0, i.z FROM small_inner i);
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q13.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q13.benchmark
new file mode 100644
index 0000000000..296b031c4f
--- /dev/null
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q13.benchmark
@@ -0,0 +1,39 @@
+name Q13
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+# Correctness canary: the NOT IN result must match a reference count that does
+# not use NOT IN. It holds for every NAJ_ROWS. `k` is NOT NULL on both sides, so
+# a subquery row makes an outer tuple UNKNOWN or TRUE exactly when `k` matches
+# and `id_n50` is NULL on either side or equal; the row is kept only when no
+# such subquery row exists.
+assert I
+SELECT count(*) = (
+ SELECT count(*)
+ FROM small_outer o
+ WHERE NOT EXISTS (
+ SELECT 1 FROM small_inner i
+ WHERE i.k = o.k
+ AND (o.id_n50 IS NULL OR i.id_n50 IS NULL OR i.id_n50 = o.id_n50)
+ )
+)
+FROM small_outer o
+WHERE (o.id_n50, o.k) NOT IN (SELECT i.id_n50, i.k FROM small_inner i);
+----
+true
+
+expect_plan HashJoinExec
+
+run
+-- Q13: uncorrelated multi-column NOT IN whose NOT NULL element `k` has only
+-- 16 distinct values, with 50% NULL in the other element on both sides. `k` is
+-- a scope key, so each scope holds 1/16 of each side and most of its rows are
+-- NULL-valued: the join must not keep pairing rows whose `NOT IN` is already
+-- known to be UNKNOWN. Compare with Q11 and Q12, whose scope key `z` has 1000
+-- distinct values.
+SELECT count(*)
+FROM small_outer o
+WHERE (o.id_n50, o.k) NOT IN (SELECT i.id_n50, i.k FROM small_inner i);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q14.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q14.benchmark
new file mode 100644
index 0000000000..3547d3c939
--- /dev/null
+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q14.benchmark
@@ -0,0 +1,45 @@
+name Q14
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+# Correctness canary: the NOT IN result must match a reference count that does
+# not use NOT IN. It holds for every NAJ_ROWS. `hk` is NOT NULL on both sides, so
+# a subquery row makes an outer tuple UNKNOWN or TRUE exactly when `hk` matches
+# and `id_n50` is NULL on either side or equal; the row is kept only when no
+# such subquery row exists.
+assert I
+SELECT count(*) = (
+ SELECT count(*)
+ FROM small_outer o
+ WHERE NOT EXISTS (
+ SELECT 1 FROM small_inner i
+ WHERE i.z < 500
+ AND i.hk = o.hk
+ AND (o.id_n50 IS NULL OR i.id_n50 IS NULL OR i.id_n50 = o.id_n50)
+ )
+)
+FROM small_outer o
+WHERE (o.id_n50, o.hk) NOT IN (
+ SELECT i.id_n50, i.hk FROM small_inner i WHERE i.z < 500
+);
+----
+true
+
+expect_plan HashJoinExec
+
+run
+-- Q14: uncorrelated multi-column NOT IN whose NOT NULL element `hk` is a scope
+-- key with one hot value among many rare ones, with 50% NULL in the other
+-- element on both sides. The NULL-valued outer rows of the hot scope each pair
+-- with its many subquery rows, while the rare ones pair with one row each: the
+-- join must not walk the hot scope once per outer row. The filter halves the
+-- subquery side, so the outer side is probed. Compare with Q13, whose scopes
+-- are all equally large.
+SELECT count(*)
+FROM small_outer o
+WHERE (o.id_n50, o.hk) NOT IN (
+ SELECT i.id_n50, i.hk FROM small_inner i WHERE i.z < 500
+);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/null_aware_join/init/load.sql b/benchmarks/sql_benchmarks/null_aware_join/init/load.sql
index 379a4f9bd3..a4cc3660d8 100644
--- a/benchmarks/sql_benchmarks/null_aware_join/init/load.sql
+++ b/benchmarks/sql_benchmarks/null_aware_join/init/load.sql
@@ -8,13 +8,14 @@
-- id_n50 Int64 NULL, 50% NULL
-- z correlation value for a non-equality correlation (`i.z < o.z`)
-- k correlation key for an equality correlation (`i.k = o.k`), 16 groups
+-- hk a hot value (-1) for 3/4 of the rows, and a rare one for the rest
--
-- `id_n0` is nullable so the planner still builds a null-aware join, but holds no
-- NULL, which is what makes it the zero-NULL baseline for the correlated queries.
-- `id` is only ~half covered by the subquery side, so the anti join returns rows
-- rather than degenerating to an empty or full result.
--- Small tables: used by the correlated and multi-column queries (Q04-Q12), whose cost grows with
+-- Small tables: used by the correlated and multi-column queries (Q04-Q14), whose cost grows with
-- the product of the two table sizes.
CREATE TABLE small_outer AS
SELECT
@@ -23,7 +24,8 @@ SELECT
CASE WHEN value % 100 = 0 THEN NULL ELSE value END AS id_n1,
CASE WHEN value % 2 = 0 THEN NULL ELSE value END AS id_n50,
value % 1000 AS z,
- value % 16 AS k
+ value % 16 AS k,
+ CASE WHEN value % 4 = 0 THEN value ELSE -1 END AS hk
FROM range(0, ${NAJ_ROWS:-10000});
CREATE TABLE small_inner AS
@@ -33,7 +35,8 @@ SELECT
CASE WHEN value % 100 = 0 THEN NULL ELSE value * 2 END AS id_n1,
CASE WHEN value % 2 = 0 THEN NULL ELSE value * 2 END AS id_n50,
value % 1000 AS z,
- value % 16 AS k
+ value % 16 AS k,
+ CASE WHEN value % 4 = 0 THEN value ELSE -1 END AS hk
FROM range(0, ${NAJ_ROWS:-10000});
-- Large tables: used by the uncorrelated queries (Q01-Q03), whose cost is linear
diff --git a/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite b/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite
index 470f65a95f..49fbe3e94d 100644
--- a/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite
+++ b/benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite
@@ -8,7 +8,7 @@ short = "r"
env = "NAJ_ROWS"
default = "10000"
values = ["10000", "..."]
-help = "Rows per table for the correlated and multi-column queries (Q04-Q12). These evaluate the join filter over candidate build x probe pairs, so their cost grows with the square of this value."
+help = "Rows per table for the correlated and multi-column queries (Q04-Q14). These evaluate the join filter over candidate build x probe pairs, so their cost grows with the square of this value."
[[options]]
name = "large-rows"
diff --git a/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs b/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs
index 738a07cf02..1abf2fbb5b 100644
--- a/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs
@@ -8,6 +8,8 @@
//! null-aware hash joins, are compared with `NOT EXISTS` / `EXISTS`
//! formulations of the same three-valued logic, which do not. Random tables
//! with NULLs on both sides are swept across batch sizes and partition counts.
+//! Some columns are NOT NULL, so that tuples mix elements that can be NULL with
+//! elements that cannot, which the null-aware join reads as scope keys.
//!
//! The subquery table is sometimes much smaller than the outer table, so
//! `JoinSelection` also swaps `NOT IN` into a null-aware `RightAnti` that
@@ -31,11 +33,12 @@ async fn null_aware_join_matches_reference() {
let mut plans = NullAwarePlans::default();
for seed in 0..20 {
let mut rng = StdRng::seed_from_u64(seed);
- let outer = random_table(&mut rng, &["id", "k", "a", "b", "c"], 6);
+ let outer = random_table(&mut rng, &["id", "k", "a", "b", "c"], &["n", "m"], 6);
// Alternate between a subquery table of similar size and a much
// smaller one that `JoinSelection` moves to the build side.
let inner_rows = if seed % 2 == 0 { 6 } else { 2 };
- let inner = random_table(&mut rng, &["k", "x", "y", "z"], inner_rows);
+ let inner =
+ random_table(&mut rng, &["k", "x", "y", "z"], &["v", "w"], inner_rows);
for target_partitions in [1, 4] {
for batch_size in [1, 3, 8192] {
let config = SessionConfig::new()
@@ -129,6 +132,22 @@ fn cases() -> Vec<(String, String)> {
"o.a = i.x AND o.b = i.y AND o.c = i.z",
),
("(o.a, 1)", "i.x, i.y", "o.a = i.x AND 1 = i.y"),
+ // `o.n`, `o.m`, `i.v` and `i.w` are NOT NULL.
+ ("(o.a, o.n)", "i.x, i.v", "o.a = i.x AND o.n = i.v"),
+ ("(o.n, o.a)", "i.v, i.x", "o.n = i.v AND o.a = i.x"),
+ (
+ "(o.n, o.a, o.b)",
+ "i.v, i.x, i.y",
+ "o.n = i.v AND o.a = i.x AND o.b = i.y",
+ ),
+ (
+ "(o.n, o.a, o.m)",
+ "i.v, i.x, i.w",
+ "o.n = i.v AND o.a = i.x AND o.m = i.w",
+ ),
+ ("(o.n, o.a)", "i.y, i.x", "o.n = i.y AND o.a = i.x"),
+ ("(1, o.a)", "i.v, i.x", "1 = i.v AND o.a = i.x"),
+ ("(o.n, o.m)", "i.v, i.w", "o.n = i.v AND o.m = i.w"),
] {
let subquery = format!("SELECT {output} FROM i{correlation}");
let matched = format!(
@@ -166,14 +185,24 @@ fn cases() -> Vec<(String, String)> {
cases
}
-/// A table of nullable `Int32` columns with values in `0..3`, split into up
-/// to three partitions of fewer than `max_rows` rows each. A column named `id`
-/// holds unique row ids instead.
-fn random_table(rng: &mut StdRng, columns: &[&str], max_rows: i32) -> Arc<MemTable> {
+/// A table of `Int32` columns with values in `0..3`, split into up to three
+/// partitions of fewer than `max_rows` rows each: the `nullable` columns, then
+/// the `non_nullable` ones. A column named `id` holds unique row ids instead.
+fn random_table(
+ rng: &mut StdRng,
+ nullable: &[&str],
+ non_nullable: &[&str],
+ max_rows: i32,
+) -> Arc<MemTable> {
+ let columns = nullable
+ .iter()
+ .map(|name| (*name, true))
+ .chain(non_nullable.iter().map(|name| (*name, false)))
+ .collect::<Vec<_>>();
let schema = Arc::new(Schema::new(
columns
.iter()
- .map(|name| Field::new(*name, DataType::Int32, true))
+ .map(|(name, nullable)| Field::new(*name, DataType::Int32, *nullable))
.collect::<Vec<_>>(),
));
let null_fraction = [0.0, 0.1, 0.3][rng.random_range(0..3)];
@@ -183,13 +212,13 @@ fn random_table(rng: &mut StdRng, columns: &[&str], max_rows: i32) -> Arc<MemTab
let rows = rng.random_range(0..max_rows);
let arrays = columns
.iter()
- .map(|name| {
+ .map(|(name, nullable)| {
let values: Int32Array = if *name == "id" {
(next_id..next_id + rows).map(Some).collect()
} else {
(0..rows)
.map(|_| {
- (!rng.random_bool(null_fraction))
+ (!(*nullable && rng.random_bool(null_fraction)))
.then(|| rng.random_range(0..3))
})
.collect()
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rs
index dd1f8d1379..b98b2b97be 100644
--- a/datafusion/core/tests/sql/joins.rs
+++ b/datafusion/core/tests/sql/joins.rs
@@ -688,3 +688,89 @@ async fn asof_join_unparser_preserves_right_preselection() -> Result<()> {
}
Ok(())
}
+
+/// Registers `o(a, b)` and `i(x, y)` with the fields that
+/// `RecordBatch::try_from_iter` infers, which count only the null bits.
+fn hidden_null_context(b: ArrayRef, y: ArrayRef) -> Result<SessionContext> {
+ let outer = RecordBatch::try_from_iter(vec![
+ (
+ "a",
+ Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef,
+ ),
+ ("b", b),
+ ])?;
+ let inner = RecordBatch::try_from_iter(vec![
+ ("x", Arc::new(Int32Array::from(vec![1])) as ArrayRef),
+ ("y", y),
+ ])?;
+ assert!(!outer.schema().field_with_name("b")?.is_nullable());
+ assert!(!inner.schema().field_with_name("y")?.is_nullable());
+ let ctx = SessionContext::new();
+ ctx.register_batch("o", outer)?;
+ ctx.register_batch("i", inner)?;
+ Ok(ctx)
+}
+
+/// A dictionary with keys `keys` into `values`.
+fn dictionary(keys: Vec<i32>, values: ArrayRef) -> Result<ArrayRef> {
+ Ok(Arc::new(DictionaryArray::<Int32Type>::try_new(
+ Int32Array::from(keys),
+ values,
+ )?))
+}
+
+const HIDDEN_NULL_PROJECTED: &str = "SELECT a, (a, b) NOT IN (SELECT x, y FROM i) AS r \
+ FROM o ORDER BY a NULLS LAST";
+const HIDDEN_NULL_FILTERED: &str =
+ "SELECT a FROM o WHERE (a, b) NOT IN (SELECT x, y FROM i)";
+
+/// A dictionary key can point to a NULL value without a null bit, so the
+/// field inferred for `b` is non-nullable although `b` holds a NULL. The
+/// multi-column `NOT IN` must still see that NULL: `(1, NULL) NOT IN ((1, 2))`
+/// is NULL, so a `WHERE` drops the row.
+#[tokio::test]
+async fn multi_column_not_in_sees_null_dictionary_values() -> Result<()> {
+ let ctx = hidden_null_context(
+ dictionary(vec![0, 1], Arc::new(Int32Array::from(vec![None, Some(3)])))?,
+ dictionary(vec![0], Arc::new(Int32Array::from(vec![2])))?,
+ )?;
+ let projected = ctx.sql(HIDDEN_NULL_PROJECTED).await?.collect().await?;
+ assert_batches_eq!(
+ [
+ "+---+------+",
+ "| a | r |",
+ "+---+------+",
+ "| 1 | |",
+ "| | true |",
+ "+---+------+",
+ ],
+ &projected
+ );
+ let filtered = ctx.sql(HIDDEN_NULL_FILTERED).await?.collect().await?;
+ assert_batches_eq!(["+---+", "| a |", "+---+", "| |", "+---+"], &filtered);
+ Ok(())
+}
+
+/// Every slot of a `Null` array is NULL without a null bit, so the inferred
+/// fields of `b` and `y` are non-nullable. Each row compares a NULL `b`, so
+/// `NOT IN` is NULL for both and a `WHERE` drops both.
+#[tokio::test]
+async fn multi_column_not_in_sees_null_type_values() -> Result<()> {
+ let ctx =
+ hidden_null_context(Arc::new(NullArray::new(2)), Arc::new(NullArray::new(1)))?;
+ let projected = ctx.sql(HIDDEN_NULL_PROJECTED).await?.collect().await?;
+ assert_batches_eq!(
+ [
+ "+---+---+",
+ "| a | r |",
+ "+---+---+",
+ "| 1 | |",
+ "| | |",
+ "+---+---+",
+ ],
+ &projected
+ );
+ let filtered = ctx.sql(HIDDEN_NULL_FILTERED).await?.collect().await?;
+ assert_batches_eq!(["++", "++"], &filtered);
+ Ok(())
+}
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index 0fac69e7a0..c5b5b23ba8 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -4644,7 +4644,9 @@ pub struct Join {
/// [`Self::null_aware`] is set.
///
/// A scalar `x NOT IN (SELECT y ...)` has one value key, a multi-column
- /// `(a, b) NOT IN (SELECT x, y ...)` one per tuple element. The value keys
+ /// `(a, b) NOT IN (SELECT x, y ...)` one per tuple element that can be
+ /// NULL in scope. A tuple element that cannot be NULL on either side never
+ /// makes the comparison UNKNOWN, so it is a scope key instead. The value keys
/// are the leading equi-join keys (`on[..n]`, or the leading equality
/// conjuncts of `filter` before they are extracted into `on`); any keys
/// after them are correlation scope keys of a correlated subquery. A NULL
diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
index 95fcdac1fe..a3d25ad4b7 100644
--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
@@ -34,6 +34,7 @@ use datafusion_expr::{
exists, in_subquery, lit, not, not_exists, not_in_subquery, when,
};
+use arrow::datatypes::DataType;
use log::debug;
/// Optimizer rule for rewriting predicate(IN/EXISTS) subquery to left semi/anti joins
@@ -556,7 +557,7 @@ struct MarkJoin {
/// Are the `IN` equalities, in order, the first equalities of `join_filter`
/// that the hash join can use as keys? A null-aware hash join reads its first
/// `V` keys as the `IN` values (one per tuple element of a multi-column
-/// `IN`) and the other keys as the correlation.
+/// `IN` that can be NULL in scope) and the other keys as the correlation.
fn values_are_first_keys(
in_values: &[InValue],
join_filter: &Expr,
@@ -618,6 +619,67 @@ impl InValue {
scope_filters,
))
}
+
+ /// Can either side hold a NULL that its schema does not show? See
+ /// [`type_may_hide_null`]. `left` is the outer plan, `subquery` the plan
+ /// that `output_expr` reads, and `right_schema` the schema of the aliased
+ /// subquery.
+ ///
+ /// Each side checks the type of its key and of every column the key
+ /// reads. A cast or a function over a column that hides a NULL can turn
+ /// it into a real NULL under a type that hides none, and the schema still
+ /// declares the result non-nullable.
+ fn may_hide_null(
+ &self,
+ left: &LogicalPlan,
+ subquery: &LogicalPlan,
+ right_schema: &DFSchema,
+ ) -> Result<bool> {
+ Ok(type_may_hide_null(&self.value.get_type(left.schema())?)
+ || type_may_hide_null(&self.subquery_column.get_type(right_schema)?)
+ || reads_column_that_may_hide_null(&self.value, left)?
+ || reads_column_that_may_hide_null(&self.output_expr, subquery)?)
+ }
+}
+
+/// Does `expr` read a column of `plan` whose type may hide a NULL (see
+/// [`type_may_hide_null`])? A column is looked up in the schemas of `plan`
+/// and its inputs. A column found in none of them is assumed to hide one,
+/// which only keeps its element a value key.
+fn reads_column_that_may_hide_null(expr: &Expr, plan: &LogicalPlan) -> Result<bool> {
+ for column in expr.column_refs() {
+ let mut found = false;
+ let mut hides_null = false;
+ plan.apply(|node| {
+ if let Ok((_, field)) = node.schema().qualified_field_from_column(column) {
+ found = true;
+ hides_null |= type_may_hide_null(field.data_type());
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ if hides_null || !found {
+ return Ok(true);
+ }
+ }
+ Ok(false)
+}
+
+/// Can an array of this type hold a NULL in a field declared non-nullable?
+///
+/// An array of these types can be logically NULL at a slot without a null
+/// bit: every slot of a `Null` array, a dictionary key that points to a NULL
+/// value, a run of NULLs, or a NULL in the selected union child. The null
+/// count covers only the null bits, so a field inferred from such an array,
+/// as `RecordBatch::try_from_iter` does, is non-nullable although the array
+/// holds NULLs.
+fn type_may_hide_null(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ DataType::Null
+ | DataType::Dictionary(_, _)
+ | DataType::RunEndEncoded(_, _)
+ | DataType::Union(_, _)
+ )
}
/// Can this join key be NULL for a row inside the scope of an outer row?
@@ -644,6 +706,11 @@ impl InValue {
/// aside. Any other conjunct is assumed to let a NULL through, which keeps
/// the join null-aware.
///
+/// A volatile key, such as one built on `random()`, is evaluated once in the
+/// subquery filter and once more as the join key. The two calls can disagree,
+/// so a conjunct on one tells nothing about the other, and such a key keeps
+/// the join null-aware.
+///
/// `key_is_outer` tells on which side of the join the key is. The scope
/// filters keep their outer references, and a side of a conjunct matches an
/// outer key only if it names outer columns alone. A subquery column can have
@@ -658,6 +725,9 @@ fn key_may_be_null_in_scope(
if !nullable {
return false;
}
+ if key.is_volatile() {
+ return true;
+ }
let key = strip_casts(key);
!scope_filters
.iter()
@@ -709,9 +779,11 @@ fn strip_casts(expr: &Expr) -> &Expr {
/// Combines the `IN`/`NOT IN` equalities, one per value, with the correlation
/// predicates, if any.
///
-/// The equalities stay the leading conjuncts, in order, so that they become
-/// `on[..V]`, the key positions a null-aware hash join reads as the `NOT IN`
-/// value keys (see `HashJoinExec::null_aware`).
+/// The equalities stay the leading conjuncts, in order, so that the first `V`
+/// of them become `on[..V]`, the key positions a null-aware hash join reads as
+/// the `NOT IN` value keys (see `HashJoinExec::null_aware`). The equalities
+/// after them, if any, are on tuple elements that cannot be NULL, which the
+/// join reads as scope keys like the correlation.
fn in_join_filter(in_values: &[InValue], correlation: Option<Expr>) -> Result<Expr> {
conjunction(
in_values
@@ -940,16 +1012,19 @@ fn build_join(
None => vec![],
};
- // True if an `IN` value or the subquery column it is compared with can be
- // NULL inside the scope of an outer row.
- let mut value_may_be_null = false;
- for in_value in &in_values {
- value_may_be_null |= in_value.may_be_null_in_scope(
- left.schema(),
- sub_query_alias.schema(),
- &pull_up.correlated_filters,
- )?;
- }
+ // For each `IN` value, true if it or the subquery column it is compared
+ // with can be NULL inside the scope of an outer row.
+ let elements_may_be_null = in_values
+ .iter()
+ .map(|in_value| {
+ in_value.may_be_null_in_scope(
+ left.schema(),
+ sub_query_alias.schema(),
+ &pull_up.correlated_filters,
+ )
+ })
+ .collect::<Result<Vec<_>>>()?;
+ let value_may_be_null = elements_may_be_null.contains(&true);
// Whether this join needs null-aware (`NOT IN` three-valued) semantics.
// Decided once: the constant projection below, the mark branch and the anti
@@ -1024,6 +1099,53 @@ fn build_join(
}
let left = projected_left.as_ref().unwrap_or(left);
+ // The first `V` keys of a null-aware join are its `NOT IN` value keys.
+ //
+ // A tuple element that cannot be NULL in scope on either side never makes
+ // the tuple comparison UNKNOWN: compared with a definite value it is either
+ // equal, which leaves the other elements to decide, or a definite mismatch,
+ // which makes the comparison FALSE whatever the other elements are. It
+ // therefore only selects which subquery rows a NULL in another element is
+ // compared with, which is what a correlation scope key does. Moving such
+ // elements after the ones that can be NULL turns them into scope keys, so
+ // the hash join finds the rows a NULL-valued row is compared with through a
+ // hash lookup instead of pairing it with every row of the other side
+ // (<https://github.com/apache/datafusion/issues/26088>).
+ //
+ // A null-aware tuple has at least one element that can be NULL, so `V` is
+ // at least one. With exactly one, the join is a scalar `NOT IN` on that
+ // element, correlated by the other elements.
+ //
+ // An element whose type can hide a NULL from its schema (see
+ // `type_may_hide_null`) stays a value key too. As a scope key, such a NULL
+ // would put its row outside every scope, which makes the comparison FALSE
+ // instead of UNKNOWN; a value key still sees it.
+ let null_aware_value_keys = if null_aware && in_values.len() > 1 {
+ let mut keyed = in_values
+ .into_iter()
+ .zip(elements_may_be_null)
+ .map(|(in_value, may_be_null)| {
+ let keeps_value_key = may_be_null
+ || in_value.may_hide_null(
+ left,
+ subquery,
+ sub_query_alias.schema(),
+ )?;
+ Ok((keeps_value_key, in_value))
+ })
+ .collect::<Result<Vec<_>>>()?;
+ // A stable sort keeps the order within each group.
+ keyed.sort_by_key(|(keeps_value_key, _)| !keeps_value_key);
+ let null_aware_value_keys = keyed
+ .iter()
+ .take_while(|(keeps_value_key, _)| *keeps_value_key)
+ .count();
+ in_values = keyed.into_iter().map(|(_, in_value)| in_value).collect();
+ null_aware_value_keys
+ } else {
+ in_values.len().max(1)
+ };
+
let join_filter = match (in_values.is_empty(), join_filter_opt) {
(false, correlation_opt) => in_join_filter(&in_values, correlation_opt)?,
(true, Some(correlation)) => correlation,
@@ -1033,23 +1155,22 @@ fn build_join(
// Only a null-aware join reads the key order.
let value_is_first_key = null_aware
&& values_are_first_keys(
- &in_values,
+ &in_values[..null_aware_value_keys],
&join_filter,
left.schema(),
sub_query_alias.schema(),
)?;
// A multi-column `IN` has no fallback that materializes the UNKNOWN rows
- // (see below), so a tuple element that does not become its key, for
- // example because its type is not hashable, is rejected instead of being
- // read in the wrong key position.
+ // (see below), so a tuple element that can be NULL and does not become its
+ // key, for example because its type is not hashable, is rejected instead
+ // of being read in the wrong key position. The other elements need not be
+ // keys: like any correlation, they may stay in the join filter.
if null_aware && !value_is_first_key && in_values.len() > 1 {
return not_impl_err!(
"Multi-column NOT IN subquery is not supported when a tuple element is not a hashable equi-join key: {join_filter}"
);
}
- // The first `V` keys of a null-aware join are its `NOT IN` value keys.
- let null_aware_value_keys = in_values.len().max(1);
// A `NOT IN` in a filter builds a `LeftAnti` join. The null-aware hash
// join reads its first key as the `NOT IN` value and the other keys as the
@@ -1317,8 +1438,8 @@ mod tests {
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_expr::builder::table_source;
use datafusion_expr::{
- TableScanBuilder, and, binary_expr, col, cube, grouping_set, out_ref_col, rollup,
- table_scan,
+ ScalarUDF, TableScanBuilder, Volatility, and, binary_expr, col, cube,
+ grouping_set, out_ref_col, rollup, table_scan,
};
use datafusion_functions_aggregate::count::count_udaf;
@@ -2460,6 +2581,39 @@ mod tests {
Ok(())
}
+ /// A volatile key is evaluated again for the join, so a correlation on the
+ /// same expression says nothing about the join key, and the mark join
+ /// stays null-aware.
+ #[test]
+ fn mark_join_for_volatile_in_predicate_correlation_is_null_aware() -> Result<()> {
+ let volatile_key = |arg: Expr| {
+ ScalarUDF::from(
+ udfs::PlacementTestUDF::new().with_volatility(Volatility::Volatile),
+ )
+ .call(vec![arg])
+ };
+ let subquery = Arc::new(
+ LogicalPlanBuilder::from(test_table_scan_with_name("sq")?)
+ .filter(
+ col("sq.c").eq(volatile_key(out_ref_col(DataType::UInt32, "test.c"))),
+ )?
+ .project(vec![col("sq.c")])?
+ .build()?,
+ );
+ let plan = LogicalPlanBuilder::from(test_table_scan()?)
+ .filter(not_in_subquery(volatile_key(col("test.c")), subquery).is_null())?
+ .build()?;
+
+ let optimized = optimize_with_decorrelate(plan)?;
+ assert!(
+ has_null_aware_left_mark_join(&optimized),
+ "{}",
+ optimized.display_indent_schema()
+ );
+
+ Ok(())
+ }
+
/// A grouping set above the correlation puts a NULL back into the key: the
/// grand-total row of `ROLLUP` is a NULL for every outer row. The
/// correlation then no longer bounds the subquery result, and the mark
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 20a7def8f3..9410280e4f 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -851,16 +851,18 @@ pub struct HashJoinExec {
/// [`Self::null_aware`] is set.
///
/// A scalar `x NOT IN (SELECT y ...)` has one value key; a multi-column
- /// `(a, b) NOT IN (SELECT x, y ...)` has one per tuple element. The tuple
- /// comparison is UNKNOWN rather than FALSE when no element pair is a
- /// definite mismatch and at least one of them involves a NULL, so a NULL in
- /// one element does not make every comparison UNKNOWN the way a scalar
- /// NULL does.
+ /// `(a, b) NOT IN (SELECT x, y ...)` has one per tuple element that can be
+ /// NULL. The tuple comparison is UNKNOWN rather than FALSE when no element
+ /// pair is a definite mismatch and at least one of them involves a NULL,
+ /// so a NULL in one element does not make every comparison UNKNOWN the
+ /// way a scalar NULL does. A tuple element that cannot be NULL on either
+ /// side never makes the comparison UNKNOWN; the planner passes it as a
+ /// scope key after the value keys, so it narrows the rows a NULL-valued
+ /// row is compared with.
///
/// Such a join decides UNKNOWN per row, like a correlated scalar `NOT IN`:
- /// without correlation keys every NULL-valued row is paired with every row
- /// on the other side to check its other elements. See
- /// <https://github.com/apache/datafusion/issues/26088> for the follow-up.
+ /// without scope keys every NULL-valued row is paired with every row on
+ /// the other side to check its other elements.
pub null_aware_value_keys: usize,
/// Cache holding plan properties like equivalences, output partitioning etc.
cache: Arc<PlanProperties>,
@@ -3302,7 +3304,6 @@ fn build_in_memory(
.map(|not_in| {
NotInBuild::try_new(
not_in,
- on_left,
&batch,
&left_values,
&normalized_values,
@@ -10582,6 +10583,111 @@ mod tests {
Ok(())
}
+ /// Runs a null-aware `LeftAnti` join with the value key `id` and the
+ /// scope key `grp`, over 10,000 build rows `(2 * i, i % 100)`, with a
+ /// NULL `id` for each multiple of `null_build_every`, and the probe rows
+ /// `(probe_ids[i], i)`, in a memory pool of `memory_limit` bytes or an
+ /// unbounded one. Returns the output rows and the `build_mem_used` metric.
+ async fn run_scope_key_anti_join(
+ null_build_every: Option<i32>,
+ probe_ids: Vec<Option<i32>>,
+ memory_limit: Option<usize>,
+ ) -> Result<(usize, usize)> {
+ let build_rows = 10_000;
+ let build_id = |i: i32| match null_build_every {
+ Some(every) if i % every == 0 => None,
+ _ => Some(i * 2),
+ };
+ let left = build_table_two_cols(
+ ("id", &(0..build_rows).map(build_id).collect()),
+ ("grp", &(0..build_rows).map(|i| Some(i % 100)).collect()),
+ );
+ let probe_grps = (0..probe_ids.len() as i32).map(Some).collect();
+ let right = build_table_two_cols(("id", &probe_ids), ("grp", &probe_grps));
+ let on = vec![
+ (
+ Arc::new(Column::new_with_schema("id", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("id", &right.schema())?) as _,
+ ),
+ (
+ Arc::new(Column::new_with_schema("grp", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("grp", &right.schema())?) as _,
+ ),
+ ];
+ let join = HashJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ &JoinType::LeftAnti,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ true,
+ )?;
+ let task_ctx = fallback_context(memory_limit);
+ let batches = common::collect(join.execute(0, task_ctx)?).await?;
+ let rows = batches.iter().map(|b| b.num_rows()).sum();
+ let build_mem_used = join
+ .metrics()
+ .unwrap()
+ .sum_by_name("build_mem_used")
+ .unwrap()
+ .as_usize();
+ Ok((rows, build_mem_used))
+ }
+
+ /// The hash table over the build rows' correlation scope keys is only
+ /// looked up for probe rows with a NULL value key, so a probe side
+ /// without one never builds it. A memory pool that holds the build side
+ /// but not that table makes the join pair the NULL-valued probe rows with
+ /// every build row instead. The result is the same in all three runs.
+ #[tokio::test]
+ async fn null_aware_scope_map_built_only_for_null_probe_values() -> Result<()> {
+ // `(0, 0)` matches the build row `(0, 0)`; `(1, 1)` matches nothing.
+ let (rows, without_null) =
+ run_scope_key_anti_join(None, vec![Some(0), Some(1)], None).await?;
+ assert_eq!(rows, 9_999);
+ // `(NULL, 1)` makes the 100 build rows with `grp = 1` UNKNOWN.
+ let (rows, with_null) =
+ run_scope_key_anti_join(None, vec![Some(0), None], None).await?;
+ assert_eq!(rows, 9_899);
+ // The scope map holds a bucket and a chain entry for every build row.
+ assert!(
+ with_null >= without_null + 10_000 * size_of::<u32>(),
+ "{without_null} -> {with_null}"
+ );
+ // Not enough memory for the scope map.
+ let limit = without_null + 10_000 * size_of::<u32>() / 2;
+ let (rows, build_mem_used) =
+ run_scope_key_anti_join(None, vec![Some(0), None], Some(limit)).await?;
+ assert_eq!(rows, 9_899, "fallback without the scope map");
+ assert_eq!(
+ build_mem_used, without_null,
+ "fallback without the scope map"
+ );
+ Ok(())
+ }
+
+ /// The same for the hash table over the scope keys of the NULL-valued
+ /// build rows, which is built with the build side.
+ #[tokio::test]
+ async fn null_aware_null_build_scope_map_falls_back_when_memory_is_short()
+ -> Result<()> {
+ // The build rows `(NULL, 0)` for each multiple of 100 are UNKNOWN for
+ // the probe row `(0, 0)`, and `(0, 0)` matches the build row `(0, 0)`
+ // no more, as its `id` is NULL.
+ let probe_ids = vec![Some(0), Some(1)];
+ let (rows, unbounded) =
+ run_scope_key_anti_join(Some(100), probe_ids.clone(), None).await?;
+ assert_eq!(rows, 9_900);
+ let (rows, build_mem_used) =
+ run_scope_key_anti_join(Some(100), probe_ids, Some(unbounded - 1)).await?;
+ assert_eq!(rows, 9_900);
+ assert!(build_mem_used < unbounded, "{build_mem_used} < {unbounded}");
+ Ok(())
+ }
+
/// `left.z > right.z` over the second column of two-column tables: the
/// non-equality correlation of
/// `id NOT IN (SELECT r.id FROM r WHERE r.z < l.z)`.
@@ -10914,6 +11020,57 @@ mod tests {
Ok(())
}
+ /// A probe batch with more NULL-valued rows than the batch size, spread
+ /// among other rows. The NULL-valued rows past the first batch of them
+ /// decide `(3, 7)` and `(4, 9)`, so the pairing must find them again at
+ /// their own rows for every chunk of build rows that the first batch
+ /// leaves undecided.
+ #[apply(hash_join_exec_configs)]
+ #[tokio::test]
+ async fn test_null_aware_left_anti_multi_column_null_probe_rows_past_batch(
+ batch_size: usize,
+ ) -> Result<()> {
+ let task_ctx = prepare_task_ctx(batch_size, false);
+ let left = build_table_two_cols(
+ ("a", &vec![Some(3), Some(4), Some(6)]),
+ ("b", &vec![Some(7), Some(9), Some(6)]),
+ );
+ let right = build_table_two_cols(
+ ("a", &vec![Some(1), None, None, Some(5), None, None]),
+ (
+ "b",
+ &vec![Some(1), Some(8), Some(8), Some(5), Some(7), Some(9)],
+ ),
+ );
+ let on = ["a", "b"]
+ .into_iter()
+ .map(|name| {
+ Ok((
+ Arc::new(Column::new_with_schema(name, &left.schema())?) as _,
+ Arc::new(Column::new_with_schema(name, &right.schema())?) as _,
+ ))
+ })
+ .collect::<Result<JoinOn>>()?;
+ // Both elements are value keys, so the NULL-valued probe rows are
+ // paired with every build row.
+ let join = HashJoinExecBuilder::new(left, right, on, JoinType::LeftAnti)
+ .with_partition_mode(PartitionMode::CollectLeft)
+ .with_null_aware(true)
+ .with_null_aware_value_keys(2)
+ .build()?;
+ let batches = common::collect(join.execute(0, task_ctx)?).await?;
+ allow_duplicates! {
+ assert_snapshot!(batches_to_sort_string(&batches), @r"
+ +---+---+
+ | a | b |
+ +---+---+
+ | 6 | 6 |
+ +---+---+
+ ");
+ }
+ Ok(())
+ }
+
/// Swaps a null-aware `LeftAnti` into the `RightAnti` that `JoinSelection`
/// plans, with its own build state so that both can be executed.
fn swap_for_separate_execution(
diff --git a/datafusion/physical-plan/src/joins/hash_join/not_in.rs b/datafusion/physical-plan/src/joins/hash_join/not_in.rs
index 1a810cdd0f..46c835bd42 100644
--- a/datafusion/physical-plan/src/joins/hash_join/not_in.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/not_in.rs
@@ -20,7 +20,7 @@
//! - [`NotInBuild::final_chunk`] on each chunk of build rows emitted after
//! the probe side is exhausted.
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
use parking_lot::Mutex;
@@ -30,12 +30,14 @@ use arrow::array::{
};
use arrow::buffer::{BooleanBuffer, NullBuffer};
use arrow::compute::{filter, filter_record_batch, take};
+use arrow::util::bit_iterator::BitIndexIterator;
use arrow::util::bit_util;
use datafusion_common::hash_utils::{RandomState, create_hashes};
-use datafusion_common::{JoinSide, JoinType, NullEquality, Result, plan_err};
+use datafusion_common::{
+ DataFusionError, JoinSide, JoinType, NullEquality, Result, plan_err,
+};
use datafusion_execution::memory_pool::MemoryReservation;
use datafusion_expr::{ColumnarValue, Operator};
-use datafusion_physical_expr::PhysicalExprRef;
use datafusion_physical_expr_common::datum::apply_cmp;
use crate::joins::PartitionMode;
@@ -46,7 +48,7 @@ use crate::joins::join_hash_map::{JoinHashMapType, ProbeOffset};
use crate::joins::probe_completion::ProbeSideSummary;
use crate::joins::utils::{
BuildProbeJoinMetrics, JoinFilter, apply_join_filter_to_indices,
- equal_rows_arr_with_normalized_left, matchable_join_keys, update_hash,
+ equal_rows_arr_with_normalized_left, matchable_join_keys, update_hash_from_values,
};
/// Which `NOT IN` a null-aware join computes, derived from
@@ -168,9 +170,9 @@ pub(super) enum NotInBuild {
/// One bit per build row, set once its mark is UNKNOWN; shared by
/// every probe partition, and never cleared.
unknown: SharedBitmapBuilder,
- /// Hash table over the scope keys (`on[value_keys..]`) of all build
- /// rows, probed with NULL-valued probe rows. `None` without scope keys.
- scope_map: Option<Box<dyn JoinHashMapType>>,
+ /// Hash table over the scope keys of all build rows, probed with
+ /// NULL-valued probe rows. `None` without scope keys.
+ scope_map: Option<BuildScopeMap>,
/// The build rows with a NULL value key, `None` when there are none.
null_value_rows: Option<NullValueBuildRows>,
},
@@ -187,7 +189,7 @@ pub(super) enum NotInBuild {
/// The fields are those of [`Self::BuildRowsCorrelated`].
ProbeRowsCorrelated {
value_keys: usize,
- scope_map: Option<Box<dyn JoinHashMapType>>,
+ scope_map: Option<BuildScopeMap>,
null_value_rows: Option<NullValueBuildRows>,
},
/// Uncorrelated [`NotIn::ProbeRows`].
@@ -201,10 +203,8 @@ impl NotInBuild {
/// Builds the `NOT IN` state of a collected build side and charges it to
/// `reservation`. `keys` are the build keys as evaluated, `normalized`
/// with `-0.0` rewritten (see `JoinLeftData::values`).
- #[expect(clippy::too_many_arguments)]
pub(super) fn try_new(
not_in: NotIn,
- on_left: &[PhysicalExprRef],
batch: &RecordBatch,
keys: &[ArrayRef],
normalized: &[ArrayRef],
@@ -215,8 +215,6 @@ impl NotInBuild {
let num_rows = batch.num_rows();
let correlated_row_state = |value_keys, reservation: &mut MemoryReservation| {
correlated_row_state(
- on_left,
- batch,
keys,
normalized,
value_keys,
@@ -465,7 +463,7 @@ impl NotInProbe {
mark_null_candidates_for_probe_batch(
left_data,
UnknownMarks::Build(unknown),
- scope_map.as_deref(),
+ scope_map.as_ref(),
null_value_rows.as_ref(),
probe_batch,
probe_values,
@@ -503,7 +501,7 @@ impl NotInProbe {
mark_null_candidates_for_probe_batch(
left_data,
UnknownMarks::Probe(&unknown),
- scope_map.as_deref(),
+ scope_map.as_ref(),
null_value_rows.as_ref(),
probe_batch,
probe_values,
@@ -588,42 +586,22 @@ fn retain_probe_rows(
}
/// The scope map and NULL-valued build rows of a correlated `NOT IN`.
-#[expect(clippy::too_many_arguments, clippy::type_complexity)]
fn correlated_row_state(
- on_left: &[PhysicalExprRef],
- batch: &RecordBatch,
keys: &[ArrayRef],
normalized: &[ArrayRef],
value_keys: usize,
random_state: &RandomState,
reservation: &mut MemoryReservation,
metrics: &BuildProbeJoinMetrics,
-) -> Result<(Option<Box<dyn JoinHashMapType>>, Option<NullValueBuildRows>)> {
- let num_rows = batch.num_rows();
- // `on_left[..V]` are the value keys and `on_left[V..]` the (possibly
- // empty) correlation scope keys.
- let scope_keys = &on_left[value_keys..];
- let scope_map = if scope_keys.is_empty() {
- None
- } else {
- // Scope-only NULL marking uses a HashMap (the primary join map may
- // use ArrayMap for full-key matches, but scope keys have arbitrary
- // shape).
- let mut scope_map = new_join_hashmap(num_rows, reservation, metrics)?;
- let mut hashes_buffer = vec![0; num_rows];
- update_hash(
- scope_keys,
- batch,
- &mut *scope_map,
- 0,
- random_state,
- &mut hashes_buffer,
- 0,
- true,
- NullEquality::NullEqualsNothing,
- )?;
- Some(scope_map)
- };
+) -> Result<(Option<BuildScopeMap>, Option<NullValueBuildRows>)> {
+ // `keys[..V]` are the value keys and `keys[V..]` the (possibly empty)
+ // correlation scope keys.
+ let has_scope_keys = keys.len() > value_keys;
+ let scope_map = has_scope_keys.then(|| BuildScopeMap {
+ map: OnceLock::new(),
+ reservation: Mutex::new(reservation.new_empty()),
+ metrics: metrics.clone(),
+ });
let Some(null_mask) = null_value_key_mask(&keys[..value_keys]) else {
return Ok((scope_map, None));
@@ -636,29 +614,101 @@ fn correlated_row_state(
reservation.try_grow(retained_size)?;
metrics.build_mem_used.add(retained_size);
- let null_scope_map = if scope_keys.is_empty() {
- None
+ let scope = if !has_scope_keys {
+ NullValueScope::Unscoped
} else {
- let scope_values = normalized[value_keys..]
- .iter()
- .map(|values| Ok(filter(values.as_ref(), &null_mask)?))
- .collect::<Result<Vec<_>>>()?;
let null_rows = build_indices.len();
- let mut map = new_join_hashmap(null_rows, reservation, metrics)?;
- let mut hashes_buffer = vec![0; null_rows];
- create_hashes(&scope_values, random_state, &mut hashes_buffer)?;
- map.update_from_iter(Box::new(hashes_buffer.iter().enumerate().rev()), 0);
- Some(map)
+ match new_join_hashmap(null_rows, reservation, metrics) {
+ Ok(mut map) => {
+ let scope_values = normalized[value_keys..]
+ .iter()
+ .map(|values| Ok(filter(values.as_ref(), &null_mask)?))
+ .collect::<Result<Vec<_>>>()?;
+ let mut hashes_buffer = vec![0; null_rows];
+ create_hashes(&scope_values, random_state, &mut hashes_buffer)?;
+ map.update_from_iter(Box::new(hashes_buffer.iter().enumerate().rev()), 0);
+ NullValueScope::Map(map)
+ }
+ // The map only speeds up the pairing, so the join goes on without
+ // it rather than fail.
+ Err(DataFusionError::ResourcesExhausted(_)) => NullValueScope::Unindexed,
+ Err(e) => return Err(e),
+ }
};
Ok((
scope_map,
Some(NullValueBuildRows {
- scope_map: null_scope_map,
+ scope,
build_indices,
}),
))
}
+/// Hash table over the correlation scope keys (`on[value_keys..]`) of all
+/// build rows, used by correlated `NOT IN` (see [`NotInScope::Correlated`]).
+///
+/// Only the probe rows with a NULL value key look it up, so it is built on
+/// the first such probe row rather than with the build side: a probe side
+/// without NULLs never pays for a second hash table over the build side.
+///
+/// The map only speeds up the pairing of those probe rows with the build
+/// rows in their scope. So when the memory pool cannot hold it, the join
+/// pairs them with every build row instead, and compares their scope keys.
+/// That is the pairing of a join whose tuple elements are all value keys,
+/// which needs no map either. The next probe batch with such rows tries to
+/// build the map again.
+pub(super) struct BuildScopeMap {
+ /// Set once the memory pool holds the map. Until then each use tries to
+ /// reserve it again, as the pool may have freed memory since.
+ map: OnceLock<Box<dyn JoinHashMapType>>,
+ /// Charged for the map once it is built. Its lock also makes the probe
+ /// partitions build the map only once.
+ reservation: Mutex<MemoryReservation>,
+ metrics: BuildProbeJoinMetrics,
+}
+
+impl BuildScopeMap {
+ /// Returns the map, building it on first use from `scope_values`, the
+ /// build side's scope keys (`JoinLeftData::values()[value_keys..]`).
+ /// Returns `None` when the memory pool cannot hold it yet.
+ fn get_or_try_init(
+ &self,
+ scope_values: &[ArrayRef],
+ random_state: &RandomState,
+ ) -> Result<Option<&dyn JoinHashMapType>> {
+ if let Some(map) = self.map.get() {
+ return Ok(Some(map.as_ref()));
+ }
+ let mut reservation = self.reservation.lock();
+ if let Some(map) = self.map.get() {
+ return Ok(Some(map.as_ref()));
+ }
+ // Scope-only NULL marking uses a HashMap (the primary join map may
+ // use ArrayMap for full-key matches, but scope keys have arbitrary
+ // shape).
+ let num_rows = scope_values[0].len();
+ let mut map = match new_join_hashmap(num_rows, &mut reservation, &self.metrics) {
+ Ok(map) => map,
+ Err(DataFusionError::ResourcesExhausted(_)) => return Ok(None),
+ Err(e) => return Err(e),
+ };
+ let mut hashes_buffer = vec![0; num_rows];
+ // A build row whose scope keys hold a NULL is in no scope, so it is
+ // left out of the map.
+ update_hash_from_values(
+ scope_values,
+ &mut *map,
+ 0,
+ random_state,
+ &mut hashes_buffer,
+ 0,
+ true,
+ NullEquality::NullEqualsNothing,
+ )?;
+ Ok(Some(self.map.get_or_init(|| map).as_ref()))
+ }
+}
+
/// The build rows with a NULL `NOT IN` value key (in any element of a
/// multi-column key), used by correlated `NOT IN` (see [`NotInScope::Correlated`]).
///
@@ -669,16 +719,26 @@ fn correlated_row_state(
/// proportional to the number of NULLs instead of enumerating every scope
/// match of every probe row.
pub(super) struct NullValueBuildRows {
- /// Hash table keyed by the correlation scope values of the NULL-valued
- /// build rows. Stored positions index into `build_indices`, not the full
- /// build batch. `None` when the join has no correlation scope keys, so
- /// every probe row is in scope.
- scope_map: Option<Box<dyn JoinHashMapType>>,
- /// Maps positions in `scope_map` back to row indices in the full build
- /// batch.
+ scope: NullValueScope,
+ /// The NULL-valued build rows. The positions stored in a
+ /// [`NullValueScope::Map`] index into it.
build_indices: UInt64Array,
}
+/// How the NULL-valued build rows find the probe rows in their correlation
+/// scope.
+enum NullValueScope {
+ /// The join has no correlation scope keys, so every probe row is in scope.
+ Unscoped,
+ /// Hash table keyed by the correlation scope values of the NULL-valued
+ /// build rows. Stored positions index into
+ /// [`NullValueBuildRows::build_indices`], not the full build batch.
+ Map(Box<dyn JoinHashMapType>),
+ /// The memory pool could not hold the map. Every probe row is paired, and
+ /// the scope keys of each pair are compared (see [`BuildScopeMap`]).
+ Unindexed,
+}
+
/// Returns the rows where at least one of `value_keys` is NULL, or `None` when
/// there is no such row.
fn null_value_key_mask(value_keys: &[ArrayRef]) -> Option<BooleanArray> {
@@ -750,7 +810,7 @@ impl<'a> UnknownMarks<'a> {
fn mark_null_candidates_for_probe_batch(
left_data: &JoinLeftData,
marks: UnknownMarks<'_>,
- scope_map: Option<&dyn JoinHashMapType>,
+ scope_map: Option<&BuildScopeMap>,
null_value_build_rows: Option<&NullValueBuildRows>,
probe_batch: &RecordBatch,
probe_values: &[ArrayRef],
@@ -850,9 +910,10 @@ fn mark_null_candidates_for_probe_batch(
// Case 1: NULL-valued build rows paired with the probe rows in their
// correlation scope.
if let Some(null_rows) = null_value_build_rows {
- let probe_rows = (0..probe_batch.num_rows() as u32).filter(in_scope);
- match &null_rows.scope_map {
- Some(scope_map) => {
+ let probe_rows =
+ |start: u32| (start..probe_batch.num_rows() as u32).filter(in_scope);
+ match &null_rows.scope {
+ NullValueScope::Map(scope_map) => {
let mut pairs =
UnmarkedPairs::new(marks, batch_size, &mut mark_scope_matches);
// The map indexes only the NULL-valued build rows; translate
@@ -860,13 +921,13 @@ fn mark_null_candidates_for_probe_batch(
pairs.scope_matches(
scope_map.as_ref(),
|position| null_rows.build_indices.value(position as usize),
- probe_rows,
+ probe_rows(0),
hashes_buffer,
lookup,
)?;
pairs.finish()?;
}
- None => {
+ NullValueScope::Unscoped => {
let mut pairs = UnmarkedPairs::new(marks, batch_size, &mark);
pairs.cross_product(
null_rows.build_indices.values().iter().copied(),
@@ -874,29 +935,55 @@ fn mark_null_candidates_for_probe_batch(
)?;
pairs.finish()?;
}
+ NullValueScope::Unindexed => {
+ let mut pairs =
+ UnmarkedPairs::new(marks, batch_size, &mut mark_scope_matches);
+ pairs.cross_product(
+ null_rows.build_indices.values().iter().copied(),
+ probe_rows,
+ )?;
+ pairs.finish()?;
+ }
}
}
// Case 2: NULL-valued probe rows paired with the build rows in their
// correlation scope.
if let Some(null_mask) = probe_null_value_mask {
- let null_probe_rows = null_mask
- .values()
- .set_indices()
- .map(|i| i as u32)
- .filter(in_scope);
+ let null_probe_rows = |start: u32| {
+ let values = null_mask.values();
+ let start = start as usize;
+ BitIndexIterator::new(
+ values.values(),
+ values.offset() + start,
+ values.len() - start,
+ )
+ .map(move |i| (start + i) as u32)
+ .filter(in_scope)
+ };
match scope_map {
+ // No NULL-valued probe row is in any scope: nothing to look up,
+ // so the scope map need not be built.
+ Some(_) if null_probe_rows(0).next().is_none() => {}
Some(scope_map) => {
let mut pairs =
UnmarkedPairs::new(marks, batch_size, &mut mark_scope_matches);
- pairs.scope_matches(
- scope_map,
- |build_row| build_row,
- null_probe_rows,
- hashes_buffer,
- lookup,
- )?;
+ match scope_map.get_or_try_init(build_scope_values, random_state)? {
+ Some(scope_map) => pairs.scope_matches(
+ scope_map,
+ |build_row| build_row,
+ null_probe_rows(0),
+ hashes_buffer,
+ lookup,
+ )?,
+ // Without the map, every build row is paired, and the
+ // scope keys of each pair are compared.
+ None => pairs.cross_product(
+ 0..left_data.batch().num_rows() as u64,
+ null_probe_rows,
+ )?,
+ }
pairs.finish()?;
}
None => {
@@ -1013,6 +1100,9 @@ struct UnmarkedPairs<'a, F> {
/// The lookups that checked the build row marks.
#[cfg(test)]
mark_checks: usize,
+ /// The most rows that one [`Self::pair_unmarked`] decided at once.
+ #[cfg(test)]
+ max_decided_rows: usize,
f: F,
}
@@ -1071,6 +1161,19 @@ fn splitmix64(x: u64) -> u64 {
x ^ (x >> 31)
}
+/// `rows` paired with their `hashes`, sorted by hash.
+fn sort_by_hash(hashes: &[u64], rows: &[u32]) -> Vec<(u64, u32)> {
+ let mut sorted: Vec<(u64, u32)> =
+ hashes.iter().copied().zip(rows.iter().copied()).collect();
+ sorted.sort_unstable_by_key(|(hash, _)| *hash);
+ sorted
+}
+
+/// The groups of equal hashes in `sorted`, sorted by hash.
+fn hash_groups(sorted: &[(u64, u32)]) -> Vec<&[(u64, u32)]> {
+ sorted.chunk_by(|a, b| a.0 == b.0).collect()
+}
+
/// Whether `rows` probe rows in `groups` groups of equal hashes save work when
/// each group is paired at once (see [`UnmarkedPairs::scope_matches`]).
/// `matches_per_row` estimates the build rows of a hash.
@@ -1123,25 +1226,78 @@ where
build_lookups: 0,
#[cfg(test)]
mark_checks: 0,
+ #[cfg(test)]
+ max_decided_rows: 0,
f,
}
}
- /// Pairs each of `probe_rows` with each of `build_rows`. The pairing stops
- /// when every row that the pairs decide is marked.
- fn cross_product(
+ /// Pairs each of the rows that `probe_rows(0)` yields with each of
+ /// `build_rows`. `probe_rows(start)` yields the probe rows from `start` on,
+ /// in increasing order. The pairing stops when every row that the pairs
+ /// decide is marked.
+ ///
+ /// When the pairs decide the build rows, the unmarked build rows are
+ /// paired in chunks of `batch_size` rows. Neither side is then held
+ /// whole: a large build side, as when this pairing stands in for a scope
+ /// map that the memory pool declined, or a probe batch larger than the
+ /// batch size. The first `batch_size` probe rows are kept for every
+ /// chunk. The rest are produced again from the first of them, only for a
+ /// chunk that still has unmarked rows after the kept ones, which then
+ /// costs less than pairing them with the chunk. A chunk is filled with
+ /// unmarked rows only, so a build side with few unmarked rows left pairs
+ /// them all at once, as a single chunk would.
+ fn cross_product<P: Iterator<Item = u32>>(
&mut self,
build_rows: impl Iterator<Item = u64>,
- probe_rows: impl Iterator<Item = u32>,
+ probe_rows: impl Fn(u32) -> P,
) -> Result<()> {
- // Drop the rows that are already marked before the first pair.
- self.needs_refresh = true;
match self.marks {
- UnknownMarks::Build(_) => {
- self.pair_unmarked(&mut build_rows.collect(), probe_rows, |b, p| (b, p))
+ UnknownMarks::Build(bitmap) => {
+ // The probe rows can be spread over a large probe batch, so
+ // keep the first ones rather than walk the batch for every
+ // chunk. The first ones often mark the whole chunk.
+ let mut kept: Vec<u32> =
+ probe_rows(0).take(self.batch_size + 1).collect();
+ if kept.is_empty() {
+ return Ok(());
+ }
+ // The first probe row past the kept ones, if any.
+ let rest_start = if kept.len() > self.batch_size {
+ kept.pop()
+ } else {
+ None
+ };
+ let mut build_rows = build_rows.peekable();
+ let mut chunk = Vec::with_capacity(self.batch_size);
+ while build_rows.peek().is_some() {
+ chunk.clear();
+ {
+ let bitmap = bitmap.lock();
+ chunk.extend(
+ build_rows
+ .by_ref()
+ .filter(|row| !bitmap.get_bit(*row as usize))
+ .take(self.batch_size),
+ );
+ }
+ // The chunk holds no row marked so far.
+ self.needs_refresh = false;
+ self.pair_unmarked(&mut chunk, kept.iter().copied(), |b, p| (b, p))?;
+ if let Some(start) = rest_start
+ && self.refresh(&mut chunk)
+ {
+ self.pair_unmarked(&mut chunk, probe_rows(start), |b, p| (b, p))?;
+ }
+ }
+ Ok(())
}
UnknownMarks::Probe(_) => {
- self.pair_unmarked(&mut probe_rows.collect(), build_rows, |p, b| (b, p))
+ // Drop the rows that are already marked before the first pair.
+ self.needs_refresh = true;
+ self.pair_unmarked(&mut probe_rows(0).collect(), build_rows, |p, b| {
+ (b, p)
+ })
}
}
}
@@ -1172,7 +1328,13 @@ where
/// Within a chunk every pair is sent, so only the later chunks can skip
/// the marked probe rows.
///
- /// Otherwise the lookup per probe row goes on.
+ /// Otherwise the lookup per probe row goes on, until it wastes as many
+ /// matches again. The estimate takes the matches per probe row so far for
+ /// every group, so a few large groups among many small ones, such as a
+ /// hot scope among rare ones, can make it miss the work that groups save.
+ /// The matches wasted again pay for a sort as well, so the join then
+ /// sorts the rest of the probe rows by hash and pairs the groups, whatever
+ /// the estimate says.
fn scope_matches(
&mut self,
scope_map: &dyn JoinHashMapType,
@@ -1232,33 +1394,48 @@ where
self.batch_size,
)
};
- let rest_hashes = &hashes[group_from..];
// Estimate the groups from a sample first, because a sort of all the
// rest that finds no group worth it costs as much as the wasted
// matches did.
- let worth_sorting = saves_work(rest_hashes.len(), estimate_groups(rest_hashes));
- let mut rest: Vec<(u64, u32)> = vec![];
- let mut groups = vec![];
- if worth_sorting {
- rest = rest_hashes
- .iter()
- .copied()
- .zip(probe_rows[group_from..].iter().copied())
- .collect();
- rest.sort_unstable_by_key(|(hash, _)| *hash);
- groups = rest.chunk_by(|a, b| a.0 == b.0).collect();
+ let mut rest = vec![];
+ if saves_work(
+ probe_rows.len() - group_from,
+ estimate_groups(&hashes[group_from..]),
+ ) {
+ rest = sort_by_hash(&hashes[group_from..], &probe_rows[group_from..]);
}
- if !worth_sorting || !saves_work(rest.len(), groups.len()) {
- self.lookup_each_row(
+ let mut groups = hash_groups(&rest);
+ if rest.is_empty() || !saves_work(rest.len(), groups.len()) {
+ // The groups are found again from where the lookups stop.
+ drop(groups);
+ self.wasted_matches = 0;
+ let Some((offset, _)) = self.lookup_each_row(
scope_map,
hashes,
(!all_rows).then_some(probe_rows.as_slice()),
offset,
&to_build_row,
- usize::MAX,
+ max_wasted_matches,
lookup,
- )?;
- return Ok(());
+ )?
+ else {
+ return Ok(());
+ };
+ let group_from = match offset {
+ AtRow { row } | MidChain { row, .. } => row,
+ };
+ if rest.is_empty() {
+ rest = sort_by_hash(&hashes[group_from..], &probe_rows[group_from..]);
+ } else {
+ // `probe_rows` is increasing, so the rows from `group_from` on
+ // are those not below its row, and keeping them keeps the
+ // order by hash.
+ match probe_rows.get(group_from) {
+ Some(&first) => rest.retain(|(_, row)| *row >= first),
+ None => rest.clear(),
+ }
+ }
+ groups = hash_groups(&rest);
}
#[cfg(test)]
@@ -1522,24 +1699,22 @@ where
/// orders a pair as (build row, probe row), and `decided_rows` are the
/// rows that the pairs decide (see [`UnknownMarks::unknown_row`]).
/// `decided_rows` must hold no row that was marked before the last chunk
- /// was sent, unless `needs_refresh` is set.
+ /// was sent, unless `needs_refresh` is set. `other_rows` is not advanced
+ /// once `decided_rows` are all marked.
fn pair_unmarked<D: Copy + Into<u64>, O: Copy>(
&mut self,
decided_rows: &mut Vec<D>,
- other_rows: impl Iterator<Item = O>,
+ mut other_rows: impl Iterator<Item = O>,
to_pair: impl Fn(D, O) -> (u64, u32),
) -> Result<()> {
- for other_row in other_rows {
- // Refresh only after a chunk was sent, so the cost of the refresh
- // stays proportional to the pairs already evaluated.
- if self.needs_refresh {
- let bitmap = self.marks.bitmap().lock();
- decided_rows.retain(|row| !bitmap.get_bit((*row).into() as usize));
- self.needs_refresh = false;
- }
- if decided_rows.is_empty() {
+ #[cfg(test)]
+ {
+ self.max_decided_rows = self.max_decided_rows.max(decided_rows.len());
+ }
+ while self.refresh(decided_rows) {
+ let Some(other_row) = other_rows.next() else {
break;
- }
+ };
for decided_row in decided_rows.iter() {
let (build_row, probe_row) = to_pair(*decided_row, other_row);
self.push(build_row, probe_row)?;
@@ -1548,6 +1723,19 @@ where
Ok(())
}
+ /// Drops the rows of `decided_rows` that are marked, if a chunk was sent
+ /// since they were last checked, and returns whether any are left.
+ fn refresh<D: Copy + Into<u64>>(&mut self, decided_rows: &mut Vec<D>) -> bool {
+ // Refresh only after a chunk was sent, so the cost of the refresh
+ // stays proportional to the pairs already evaluated.
+ if self.needs_refresh {
+ let bitmap = self.marks.bitmap().lock();
+ decided_rows.retain(|row| !bitmap.get_bit((*row).into() as usize));
+ self.needs_refresh = false;
+ }
+ !decided_rows.is_empty()
+ }
+
/// Pushes the pairs of `build_rows` and `probe_rows`.
fn push_all(&mut self, mut build_rows: &[u64], mut probe_rows: &[u32]) -> Result<()> {
while !build_rows.is_empty() {
@@ -1595,11 +1783,12 @@ mod tests {
use crate::joins::join_hash_map::JoinHashMapU64;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
+ use std::cell::Cell;
/// The input of one run of [`UnmarkedPairs`]. Each row has a scope key,
/// and the hash of a key is `key % hash_modulus`, so different scopes can
/// share a hash.
- #[derive(Debug)]
+ #[derive(Debug, Clone)]
struct Case {
mark_build_rows: bool,
/// Whether to pair through a scope map, or as a cross product.
@@ -1721,8 +1910,11 @@ mod tests {
let hash = |key: u64| key % self.hash_modulus;
let mut foreign_marks = vec![];
+ let (mut sends, mut sent_pairs) = (0, 0);
let mark = |build: UInt64Array, probe: UInt32Array| {
assert!(build.len() <= self.batch_size);
+ sends += 1;
+ sent_pairs += build.len();
if self.foreign_marks {
let row = (self.filter_seed as usize)
.wrapping_add(foreign_marks.len().wrapping_mul(0x9E37_79B9))
@@ -1749,6 +1941,7 @@ mod tests {
Ok(())
};
+ let (probe_walks, probe_reads) = (Cell::new(0), Cell::new(0));
let mut pairs = UnmarkedPairs::new(marks, self.batch_size, mark);
if self.through_scope_map {
let mut scope_map = JoinHashMapU64::with_capacity(self.build_rows.len());
@@ -1769,14 +1962,20 @@ mod tests {
&mut LookupBuffers::default(),
)?;
} else {
- pairs.cross_product(
- self.build_rows.iter().copied(),
- self.probe_rows.iter().copied(),
- )?;
+ let probe_reads = &probe_reads;
+ pairs.cross_product(self.build_rows.iter().copied(), |start| {
+ probe_walks.set(probe_walks.get() + 1);
+ let from = self.probe_rows.partition_point(|row| *row < start);
+ self.probe_rows[from..]
+ .iter()
+ .copied()
+ .inspect(move |_| probe_reads.set(probe_reads.get() + 1))
+ })?;
}
let (considered_groups, grouped) = (pairs.considered_groups, pairs.grouped);
let looked_up_matches = pairs.looked_up_matches;
let (build_lookups, mark_checks) = (pairs.build_lookups, pairs.mark_checks);
+ let max_decided_rows = pairs.max_decided_rows;
pairs.finish()?;
let bitmap = bitmap.into_inner().finish();
let marks = (0..bitmap.len()).map(|row| bitmap.value(row)).collect();
@@ -1788,6 +1987,11 @@ mod tests {
looked_up_matches,
build_lookups,
mark_checks,
+ max_decided_rows,
+ sends,
+ sent_pairs,
+ probe_walks: probe_walks.get(),
+ probe_reads: probe_reads.get(),
foreign_marks,
},
))
@@ -1801,6 +2005,14 @@ mod tests {
looked_up_matches: usize,
build_lookups: usize,
mark_checks: usize,
+ max_decided_rows: usize,
+ /// The chunks of pairs sent, and the pairs in them.
+ sends: usize,
+ sent_pairs: usize,
+ /// The calls of the `probe_rows` of a cross product, and the probe rows
+ /// they produced.
+ probe_walks: usize,
+ probe_reads: usize,
/// The rows that `foreign_marks` marked.
foreign_marks: Vec<usize>,
}
@@ -1893,6 +2105,193 @@ mod tests {
Ok(())
}
+ /// A hot scope among rare ones: `rare` probe rows of their own scope
+ /// each, then `hot` probe rows of the hot scope, then `rare` more of their
+ /// own scope. Each rare scope holds one build row, and the hot scope 7,679.
+ /// The rare rows keep the estimated groups small, but the hot group alone
+ /// saves much work, so the pairing must still switch to groups instead of
+ /// walking the hot scope once per probe row.
+ #[test]
+ fn hot_scope_among_rare_ones_is_grouped() -> Result<()> {
+ // With marks on the probe rows, the average group fills no chunk. With
+ // marks on the build rows, the groups hold fewer than two rows on
+ // average.
+ for (mark_build_rows, rare, hot) in [(false, 256, 7680), (true, 4096, 2048)] {
+ let rare_scopes = rare as u64;
+ // Scope 0 is hot.
+ let build_keys: Vec<u64> = (1..=2 * rare_scopes)
+ .chain(std::iter::repeat_n(0, 7679))
+ .collect();
+ let probe_keys: Vec<u64> = (1..=rare_scopes)
+ .chain(std::iter::repeat_n(0, hot))
+ .chain(rare_scopes + 1..=2 * rare_scopes)
+ .collect();
+ let num_marked = if mark_build_rows {
+ build_keys.len()
+ } else {
+ probe_keys.len()
+ };
+ // Every pair passes, and every scope is on both sides.
+ let case = Case {
+ mark_build_rows,
+ through_scope_map: true,
+ batch_size: 8192,
+ hash_modulus: u64::MAX,
+ probe_rows: (0..probe_keys.len() as u32).collect(),
+ build_rows: (0..build_keys.len() as u64).collect(),
+ build_keys,
+ probe_keys,
+ marked_before: vec![false; num_marked],
+ filter_seed: 0,
+ filter_one_in: 1,
+ foreign_marks: false,
+ };
+ let (marks, stats) = case.run()?;
+ assert!(marks.iter().all(|marked| *marked));
+ assert!(stats.grouped, "marks on build rows: {mark_build_rows}");
+ // The matches of two rounds of lookups per probe row, each until
+ // it wastes enough matches to pay for a sort, and a few chunks of
+ // the hot scope. The lookups that skip the check of the build row
+ // marks count no wasted match, so a round can last a little
+ // longer. A lookup per probe row walks the hot scope for each of
+ // its probe rows, tens of millions of matches.
+ let bound = 3 * WASTED_MATCHES_PER_SORTED_ROW * (2 * rare + hot) + 4 * 8192;
+ assert!(
+ stats.looked_up_matches <= bound,
+ "marks on build rows: {mark_build_rows}: {} matches walked, bound {bound}",
+ stats.looked_up_matches
+ );
+ }
+ Ok(())
+ }
+
+ /// A scope map that the memory pool declines is built on a later use,
+ /// once the pool has room for it.
+ #[test]
+ fn declined_scope_map_is_built_once_memory_is_free() -> Result<()> {
+ use crate::metrics::ExecutionPlanMetricsSet;
+ use datafusion_execution::memory_pool::{
+ GreedyMemoryPool, MemoryConsumer, MemoryPool,
+ };
+
+ let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1 << 20));
+ let other = MemoryConsumer::new("other").register(&pool);
+ other.try_grow(1 << 20)?;
+ let scope_map = BuildScopeMap {
+ map: OnceLock::new(),
+ reservation: Mutex::new(MemoryConsumer::new("scope map").register(&pool)),
+ metrics: BuildProbeJoinMetrics::new(0, &ExecutionPlanMetricsSet::new()),
+ };
+ let scope_values: Vec<ArrayRef> =
+ vec![Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3]))];
+ let random_state = RandomState::with_seed(0);
+ assert!(
+ scope_map
+ .get_or_try_init(&scope_values, &random_state)?
+ .is_none()
+ );
+ other.free();
+ assert!(
+ scope_map
+ .get_or_try_init(&scope_values, &random_state)?
+ .is_some()
+ );
+ assert!(scope_map.reservation.lock().size() > 0);
+ Ok(())
+ }
+
+ /// A cross product with marks on the build rows holds at most a batch of
+ /// build rows at once, however many build rows there are, and at most a
+ /// batch of probe rows. No pair passes, so no row is marked and every
+ /// build row is paired.
+ #[test]
+ fn cross_product_chunks_the_build_rows() -> Result<()> {
+ let (num_build, batch_size) = (10_000, 64);
+ let case = Case {
+ mark_build_rows: true,
+ through_scope_map: false,
+ batch_size,
+ hash_modulus: 1,
+ build_keys: vec![0; num_build],
+ probe_keys: vec![0; 3],
+ probe_rows: (0..3).collect(),
+ build_rows: (0..num_build as u64).collect(),
+ marked_before: vec![false; num_build],
+ filter_seed: 0,
+ filter_one_in: u64::MAX,
+ foreign_marks: false,
+ };
+ let (marks, stats) = case.run()?;
+ assert!(!marks.contains(&true));
+ assert_eq!(stats.max_decided_rows, batch_size);
+ // The three probe rows are kept for all 157 chunks.
+ let num_chunks = num_build.div_ceil(batch_size);
+ assert_eq!((stats.probe_walks, stats.probe_reads), (1, 3));
+ // A batch of probe rows is kept too, and nothing follows it.
+ let (marks, stats) = Case {
+ probe_keys: vec![0; batch_size],
+ probe_rows: (0..batch_size as u32).collect(),
+ ..case.clone()
+ }
+ .run()?;
+ assert!(!marks.contains(&true));
+ assert_eq!((stats.probe_walks, stats.probe_reads), (1, batch_size));
+ // The probe rows past the kept ones are produced again, from the
+ // first of them, for each chunk that the kept ones leave with unmarked
+ // rows.
+ let (marks, stats) = Case {
+ probe_keys: vec![0; batch_size + 1],
+ probe_rows: (0..=batch_size as u32).collect(),
+ ..case.clone()
+ }
+ .run()?;
+ assert!(!marks.contains(&true));
+ assert_eq!(
+ (stats.probe_walks, stats.probe_reads),
+ (1 + num_chunks, batch_size + 1 + num_chunks)
+ );
+ // When the kept probe rows mark each chunk, the rest are not produced
+ // again.
+ let (marks, stats) = Case {
+ probe_keys: vec![0; batch_size + 1],
+ probe_rows: (0..=batch_size as u32).collect(),
+ filter_one_in: 1,
+ ..case
+ }
+ .run()?;
+ assert!(!marks.contains(&false));
+ assert_eq!((stats.probe_walks, stats.probe_reads), (1, batch_size + 1));
+ Ok(())
+ }
+
+ /// A cross product with marks on the build rows fills its chunks with
+ /// unmarked build rows. With one unmarked build row in each batch of build
+ /// rows, all of them fit one chunk, and the pairs of the first probe row
+ /// mark them all at once.
+ #[test]
+ fn cross_product_fills_chunks_with_unmarked_build_rows() -> Result<()> {
+ let (batch_size, num_probe) = (64, 64);
+ let num_build = batch_size * batch_size;
+ let case = Case {
+ mark_build_rows: true,
+ through_scope_map: false,
+ batch_size,
+ hash_modulus: 1,
+ build_keys: vec![0; num_build],
+ probe_keys: vec![0; num_probe],
+ probe_rows: (0..num_probe as u32).collect(),
+ build_rows: (0..num_build as u64).collect(),
+ marked_before: (0..num_build).map(|row| row % batch_size != 0).collect(),
+ filter_seed: 0,
+ filter_one_in: 1,
+ foreign_marks: false,
+ };
+ let (marks, stats) = case.run()?;
+ assert!(marks.iter().all(|marked| *marked));
+ assert_eq!((stats.sends, stats.sent_pairs), (1, batch_size));
+ Ok(())
+ }
+
/// One scope holds every row, and every pair passes. Returns the matches
/// that the scope map lookups walked.
fn walked_matches_in_one_scope(
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index 55fbc2cd11..e6024d177a 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -2352,15 +2352,39 @@ pub fn update_hash(
) -> Result<()> {
// evaluate the keys
let keys_values = evaluate_expressions_to_arrays(on, batch)?;
+ update_hash_from_values(
+ &keys_values,
+ hash_map,
+ offset,
+ random_state,
+ hashes_buffer,
+ deleted_offset,
+ fifo_hashmap,
+ null_equality,
+ )
+}
+/// [`update_hash`] over the key columns `keys_values` that `on` evaluates to,
+/// with one entry of `hashes_buffer` for each of their rows.
+#[expect(clippy::too_many_arguments)]
+pub(crate) fn update_hash_from_values(
+ keys_values: &[ArrayRef],
+ hash_map: &mut dyn JoinHashMapType,
+ offset: usize,
+ random_state: &RandomState,
+ hashes_buffer: &mut [u64],
+ deleted_offset: usize,
+ fifo_hashmap: bool,
+ null_equality: NullEquality,
+) -> Result<()> {
// calculate the hash values
- let hash_values = create_hashes(&keys_values, random_state, hashes_buffer)?;
+ let hash_values = create_hashes(keys_values, random_state, hashes_buffer)?;
// For usual JoinHashmap, the implementation is void.
- hash_map.extend_zero(batch.num_rows());
+ hash_map.extend_zero(hash_values.len());
// Unmatchable NULL-key rows are filtered out below.
- let valid_keys = matchable_join_keys(&keys_values, null_equality);
+ let valid_keys = matchable_join_keys(keys_values, null_equality);
// Updating JoinHashMap from hash values iterator
let hash_values_iter = hash_values
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index 4f16c34f48..4edd948345 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -3894,7 +3894,7 @@ async fn roundtrip_join_null_aware() -> Result<()> {
(
"
SELECT k
- FROM (VALUES (1, 1, 2), (2, 3, 4)) AS t1(k, a, b)
+ FROM (VALUES (1, 1, 2), (2, 3, CAST(NULL AS INT))) AS t1(k, a, b)
WHERE (a, b) NOT IN (
SELECT x, y
FROM (VALUES (1, CAST(NULL AS INT), 2)) AS t2(k, x, y)
diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
index f28d0022f9..144e9de242 100644
--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
@@ -397,9 +397,10 @@ NULL 0
#############
# Results under a batch size sweep are in null_aware_multi_column.slt. A tuple
-# NOT IN is a null-aware join whose leading keys are the tuple elements
-# (`value_keys`), followed by any correlation keys. Like the scalar NOT IN,
-# JoinSelection swaps it into a null-aware RightAnti that builds on the
+# NOT IN is a null-aware join whose leading keys are the tuple elements that
+# can be NULL (`value_keys`), followed by the elements that cannot and by any
+# correlation keys, which only narrow the scope of a row. Like the scalar NOT
+# IN, JoinSelection swaps it into a null-aware RightAnti that builds on the
# subquery side when that side is not larger, keeping the key order.
statement ok
@@ -468,6 +469,217 @@ physical_plan
04)------DataSourceExec: partitions=1, partition_sizes=[1]
05)------DataSourceExec: partitions=1, partition_sizes=[1]
+# A tuple element that cannot be NULL on either side never makes the tuple
+# comparison UNKNOWN. It follows the elements that can be NULL as a scope key,
+# so `value_keys` counts only the nullable elements.
+statement ok
+CREATE TABLE multi_col_outer_nn(k INT NOT NULL, a INT, b INT NOT NULL) AS VALUES
+(1, 1, 2),
+(2, NULL, 8);
+
+statement ok
+CREATE TABLE multi_col_inner_nn(k INT NOT NULL, x INT, y INT NOT NULL) AS VALUES
+(1, 1, 2),
+(2, NULL, 4);
+
+# `b = y` moves after `a = x`.
+query TT
+EXPLAIN SELECT * FROM multi_col_outer_nn
+WHERE (b, a) NOT IN (SELECT y, x FROM multi_col_inner_nn);
+----
+logical_plan
+01)LeftAnti Join: multi_col_outer_nn.a = __correlated_sq_1.x, multi_col_outer_nn.b = __correlated_sq_1.y null_aware
+02)--TableScan: multi_col_outer_nn projection=[k, a, b]
+03)--SubqueryAlias: __correlated_sq_1
+04)----Projection: multi_col_inner_nn.y, multi_col_inner_nn.x
+05)------TableScan: multi_col_inner_nn projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@1, a@1), (y@0, b@2)], null_aware
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# The element scope key comes before the correlation keys.
+query TT
+EXPLAIN SELECT * FROM multi_col_outer_nn o
+WHERE (b, a) NOT IN (SELECT y, x FROM multi_col_inner_nn i WHERE i.k = o.k);
+----
+logical_plan
+01)LeftAnti Join: o.a = __correlated_sq_1.x, o.b = __correlated_sq_1.y, o.k = __correlated_sq_1.k null_aware
+02)--SubqueryAlias: o
+03)----TableScan: multi_col_outer_nn projection=[k, a, b]
+04)--SubqueryAlias: __correlated_sq_1
+05)----Projection: i.y, i.x, i.k
+06)------SubqueryAlias: i
+07)--------TableScan: multi_col_inner_nn projection=[k, x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@1, a@1), (y@0, b@2), (k@2, k@0)], null_aware
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# `NULLIF` can give a NULL, so `k` stays a value key, after `a` as in the
+# tuple, and only `b` moves.
+query TT
+EXPLAIN SELECT * FROM multi_col_outer_nn
+WHERE (b, a, k) NOT IN (SELECT y, x, NULLIF(k, 0) FROM multi_col_inner_nn);
+----
+logical_plan
+01)LeftAnti Join: multi_col_outer_nn.a = __correlated_sq_1.x, CAST(multi_col_outer_nn.k AS Int64) = __correlated_sq_1.nullif(multi_col_inner_nn.k,Int64(0)), multi_col_outer_nn.b = __correlated_sq_1.y null_aware(value_keys=2)
+02)--TableScan: multi_col_outer_nn projection=[k, a, b]
+03)--SubqueryAlias: __correlated_sq_1
+04)----Projection: multi_col_inner_nn.y, multi_col_inner_nn.x, nullif(CAST(multi_col_inner_nn.k AS Int64), Int64(0))
+05)------TableScan: multi_col_inner_nn projection=[k, x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@1, a@1), (nullif(multi_col_inner_nn.k,Int64(0))@2, CAST(multi_col_outer_nn.k AS Int64)@3), (y@0, b@2)], projection=[k@0, a@1, b@2], null_aware(value_keys=2)
+02)--ProjectionExec: expr=[y@2 as y, x@1 as x, nullif(CAST(k@0 AS Int64), 0) as nullif(multi_col_inner_nn.k,Int64(0))]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)--ProjectionExec: expr=[k@0 as k, a@1 as a, b@2 as b, CAST(k@0 AS Int64) as CAST(multi_col_outer_nn.k AS Int64)]
+05)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+# A constant element is NOT NULL, so against a NOT NULL column it is a scope
+# key.
+query TT
+EXPLAIN SELECT * FROM multi_col_outer_nn
+WHERE (2, a) NOT IN (SELECT y, x FROM multi_col_inner_nn);
+----
+logical_plan
+01)Projection: multi_col_outer_nn.k, multi_col_outer_nn.a, multi_col_outer_nn.b
+02)--LeftAnti Join: multi_col_outer_nn.a = __correlated_sq_1.x, __correlated_sq_1_value_0 = __correlated_sq_1.multi_col_inner_nn.y null_aware
+03)----Projection: multi_col_outer_nn.k, multi_col_outer_nn.a, multi_col_outer_nn.b, Int64(2) AS __correlated_sq_1_value_0
+04)------TableScan: multi_col_outer_nn projection=[k, a, b]
+05)----SubqueryAlias: __correlated_sq_1
+06)------Projection: CAST(multi_col_inner_nn.y AS Int64), multi_col_inner_nn.x
+07)--------TableScan: multi_col_inner_nn projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@1, a@1), (multi_col_inner_nn.y@0, __correlated_sq_1_value_0@3)], projection=[k@0, a@1, b@2], null_aware
+02)--ProjectionExec: expr=[CAST(y@1 AS Int64) as multi_col_inner_nn.y, x@0 as x]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)--ProjectionExec: expr=[k@0 as k, a@1 as a, b@2 as b, 2 as __correlated_sq_1_value_0]
+05)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+# NOT NULL on the outer side only: the subquery can still give a NULL `y`, so
+# `b` stays a value key.
+query TT
+EXPLAIN SELECT * FROM multi_col_outer_nn
+WHERE (b, a) NOT IN (SELECT y, x FROM multi_col_inner);
+----
+logical_plan
+01)LeftAnti Join: multi_col_outer_nn.b = __correlated_sq_1.y, multi_col_outer_nn.a = __correlated_sq_1.x null_aware(value_keys=2)
+02)--TableScan: multi_col_outer_nn projection=[k, a, b]
+03)--SubqueryAlias: __correlated_sq_1
+04)----Projection: multi_col_inner.y, multi_col_inner.x
+05)------TableScan: multi_col_inner projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(y@0, b@2), (x@1, a@1)], null_aware(value_keys=2)
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# The mark join reorders the same way.
+query TT
+EXPLAIN SELECT b, a, (b, a) NOT IN (SELECT y, x FROM multi_col_inner_nn) AS r
+FROM multi_col_outer_nn;
+----
+logical_plan
+01)Projection: multi_col_outer_nn.b, multi_col_outer_nn.a, NOT __correlated_sq_1.mark AS r
+02)--LeftMark Join: multi_col_outer_nn.a = __correlated_sq_1.x, multi_col_outer_nn.b = __correlated_sq_1.y null_aware
+03)----TableScan: multi_col_outer_nn projection=[a, b]
+04)----SubqueryAlias: __correlated_sq_1
+05)------Projection: multi_col_inner_nn.y, multi_col_inner_nn.x
+06)--------TableScan: multi_col_inner_nn projection=[x, y]
+physical_plan
+01)ProjectionExec: expr=[b@1 as b, a@0 as a, NOT mark@2 as r]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(a@0, x@1), (b@1, y@0)], null_aware
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# A correlated filter that rejects NULL makes `b = y` a scope key although
+# both columns are nullable (the `mc_scope` results in
+# null_aware_multi_column.slt).
+query TT
+EXPLAIN SELECT * FROM multi_col_outer o
+WHERE (a, b) NOT IN (SELECT x, y FROM multi_col_inner i WHERE i.y >= o.b);
+----
+logical_plan
+01)LeftAnti Join: o.a = __correlated_sq_1.x, o.b = __correlated_sq_1.y Filter: __correlated_sq_1.y >= o.b null_aware
+02)--SubqueryAlias: o
+03)----TableScan: multi_col_outer projection=[k, a, b]
+04)--SubqueryAlias: __correlated_sq_1
+05)----SubqueryAlias: i
+06)------TableScan: multi_col_inner projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (y@1, b@2)], filter=y@1 >= b@0, null_aware
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# A dictionary can hold a NULL without a null bit, so its NOT NULL field does
+# not rule out a NULL element. A cast to a common type hides the dictionary
+# from the key's type, so the type of each column the key reads counts too:
+# `b` stays a value key whichever side is cast.
+statement ok
+CREATE TABLE multi_col_outer_dict AS
+SELECT k, a, arrow_cast(b, 'Dictionary(Int32, Int32)') AS b FROM multi_col_outer_nn;
+
+statement ok
+CREATE TABLE multi_col_inner_i64 AS
+SELECT k, x, arrow_cast(y, 'Int64') AS y FROM multi_col_inner_nn;
+
+query TT
+EXPLAIN SELECT * FROM multi_col_outer_dict
+WHERE (a, b) NOT IN (SELECT x, y FROM multi_col_inner_i64);
+----
+logical_plan
+01)LeftAnti Join: multi_col_outer_dict.a = __correlated_sq_1.x, CAST(multi_col_outer_dict.b AS Int64) = __correlated_sq_1.y null_aware(value_keys=2)
+02)--TableScan: multi_col_outer_dict projection=[k, a, b]
+03)--SubqueryAlias: __correlated_sq_1
+04)----TableScan: multi_col_inner_i64 projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (y@1, CAST(multi_col_outer_dict.b AS Int64)@3)], projection=[k@0, a@1, b@2], null_aware(value_keys=2)
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--ProjectionExec: expr=[k@0 as k, a@1 as a, b@2 as b, CAST(b@2 AS Int64) as CAST(multi_col_outer_dict.b AS Int64)]
+04)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+CREATE TABLE multi_col_outer_i64 AS
+SELECT k, a, arrow_cast(b, 'Int64') AS b FROM multi_col_outer_nn;
+
+statement ok
+CREATE TABLE multi_col_inner_dict AS
+SELECT k, x, arrow_cast(y, 'Dictionary(Int32, Int32)') AS y FROM multi_col_inner_nn;
+
+query TT
+EXPLAIN SELECT * FROM multi_col_outer_i64
+WHERE (a, b) NOT IN (SELECT x, y FROM multi_col_inner_dict);
+----
+logical_plan
+01)LeftAnti Join: multi_col_outer_i64.a = __correlated_sq_1.x, multi_col_outer_i64.b = __correlated_sq_1.multi_col_inner_dict.y null_aware(value_keys=2)
+02)--TableScan: multi_col_outer_i64 projection=[k, a, b]
+03)--SubqueryAlias: __correlated_sq_1
+04)----Projection: multi_col_inner_dict.x, CAST(multi_col_inner_dict.y AS Int64)
+05)------TableScan: multi_col_inner_dict projection=[x, y]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (multi_col_inner_dict.y@1, b@2)], null_aware(value_keys=2)
+02)--ProjectionExec: expr=[x@0 as x, CAST(y@1 AS Int64) as multi_col_inner_dict.y]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+DROP TABLE multi_col_outer_dict;
+
+statement ok
+DROP TABLE multi_col_inner_i64;
+
+statement ok
+DROP TABLE multi_col_outer_i64;
+
+statement ok
+DROP TABLE multi_col_inner_dict;
+
+statement ok
+DROP TABLE multi_col_outer_nn;
+
+statement ok
+DROP TABLE multi_col_inner_nn;
+
statement ok
DROP TABLE multi_col_outer;
diff --git a/datafusion/sqllogictest/test_files/null_aware_multi_column.slt b/datafusion/sqllogictest/test_files/null_aware_multi_column.slt
index 27be58dc4c..aec2d05184 100644
--- a/datafusion/sqllogictest/test_files/null_aware_multi_column.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_multi_column.slt
@@ -533,6 +533,148 @@ z
statement ok
DEALLOCATE mc_tuple_qualified;
+#############
+## NOT NULL tuple elements
+#############
+
+# `b` and `y` cannot be NULL, so they never make the tuple comparison UNKNOWN
+# and the join reads them as a scope key: a NULL `a` is compared only with the
+# subquery rows whose `y` equals its `b`.
+statement ok
+CREATE TABLE mc_nn_outer(k INT NOT NULL, a INT, b INT NOT NULL, value TEXT) AS VALUES
+(1, 1, 2, 'p'),
+(1, NULL, 2, 'q'),
+(1, NULL, 3, 'r'),
+(1, 5, 2, 's'),
+(2, NULL, 2, 't'),
+(2, 7, 9, 'u'),
+(1, 6, 4, 'v');
+
+statement ok
+CREATE TABLE mc_nn_inner(k INT NOT NULL, x INT, y INT NOT NULL) AS VALUES
+(1, 1, 2),
+(1, NULL, 4),
+(2, 8, 9);
+
+# p matches (1, 2). q and t vs (1, 2) and v vs (NULL, 4) are UNKNOWN. r, s and
+# u mismatch every row.
+query T rowsort
+SELECT value FROM mc_nn_outer
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_nn_inner);
+----
+r
+s
+u
+
+# With a correlation, t's scope is k = 2, {(8, 9)}, which it mismatches.
+query T rowsort
+SELECT value FROM mc_nn_outer o
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_nn_inner i WHERE i.k = o.k);
+----
+r
+s
+t
+u
+
+# Two nullable elements and a scope key. The subquery gives (1, 2, 1),
+# (NULL, 4, 1) and (8, 9, NULL): t's k = 2 and u's a = 7 are definite
+# mismatches.
+query T rowsort
+SELECT value FROM mc_nn_outer
+WHERE (a, b, k) NOT IN (SELECT x, y, NULLIF(k, 2) FROM mc_nn_inner);
+----
+r
+s
+t
+u
+
+# A constant NOT NULL element: only (1, 2) is in scope.
+query T rowsort
+SELECT value FROM mc_nn_outer
+WHERE (a, 2) NOT IN (SELECT x, y FROM mc_nn_inner);
+----
+s
+u
+v
+
+# A RunEndEncoded element is not hashable. A run of NULLs has no null bit, so
+# a field declared NOT NULL does not rule out a NULL element, and the element
+# is treated like one that can be NULL: it is rejected (see above) rather than
+# read as a correlation in the join filter.
+statement ok
+CREATE TABLE mc_nn_ree_outer AS
+SELECT k, a, arrow_cast(b, 'RunEndEncoded("run_ends": non-null Int32, "values": Int32)') AS b, value
+FROM mc_nn_outer;
+
+statement ok
+CREATE TABLE mc_nn_ree_inner AS
+SELECT k, x, arrow_cast(y, 'RunEndEncoded("run_ends": non-null Int32, "values": Int32)') AS y
+FROM mc_nn_inner;
+
+statement error Multi-column NOT IN subquery is not supported when a tuple element is not a hashable equi-join key
+SELECT value FROM mc_nn_ree_outer
+WHERE (b, a) NOT IN (SELECT y, x FROM mc_nn_ree_inner);
+
+# As a nullable value.
+query TB rowsort
+SELECT value, (a, b) NOT IN (SELECT x, y FROM mc_nn_inner) AS r
+FROM mc_nn_outer;
+----
+p false
+q NULL
+r true
+s true
+t NULL
+u true
+v NULL
+
+query TB rowsort
+SELECT value, (a, b) NOT IN (SELECT x, y FROM mc_nn_inner i WHERE i.k = o.k) AS r
+FROM mc_nn_outer o;
+----
+p false
+q NULL
+r true
+s true
+t true
+u true
+v NULL
+
+# An element that can be NULL in the schema but not in scope: the correlated
+# filter `i.y >= o.b` rejects a NULL `y` or `b`, so `b = y` is a scope key, and
+# the rows with a NULL `b` reach the join with a NULL scope key. Such a row has
+# an empty subquery, so its NOT IN is TRUE.
+statement ok
+CREATE TABLE mc_scope_outer(k INT, a INT, b INT) AS VALUES
+(1, 1, 2), (2, NULL, 2), (3, NULL, NULL), (4, 5, NULL), (5, NULL, 7), (6, 1, 9), (7, 3, 3);
+
+statement ok
+CREATE TABLE mc_scope_inner(x INT, y INT) AS VALUES
+(1, 2), (NULL, 2), (5, NULL), (NULL, NULL), (3, 1), (8, 4);
+
+query IB
+SELECT k, (a, b) NOT IN (SELECT x, y FROM mc_scope_inner i WHERE i.y >= o.b)
+FROM mc_scope_outer o ORDER BY k;
+----
+1 false
+2 NULL
+3 true
+4 true
+5 true
+6 true
+7 true
+
+query I
+SELECT k FROM mc_scope_outer o
+WHERE (a, b) NOT IN (SELECT x, y FROM mc_scope_inner i WHERE i.y >= o.b)
+ORDER BY k;
+----
+3
+4
+5
+6
+7
+
#############
## Cleanup
#############
@@ -602,3 +744,21 @@ DROP TABLE mc_p1;
statement ok
DROP TABLE mc_p2;
+
+statement ok
+DROP TABLE mc_nn_outer;
+
+statement ok
+DROP TABLE mc_nn_inner;
+
+statement ok
+DROP TABLE mc_nn_ree_outer;
+
+statement ok
+DROP TABLE mc_nn_ree_inner;
+
+statement ok
+DROP TABLE mc_scope_outer;
+
+statement ok
+DROP TABLE mc_scope_inner;
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 2b11283bd0..130aa6f23a 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1015,9 +1015,17 @@ Wire compatibility is directional:
### Null-aware joins record their `NOT IN` value key count
Multi-column `(a, b) NOT IN (SELECT x, y ...)` is now planned as a null-aware
-join. Its first equi-join keys are the tuple elements, followed by any
-correlation keys, so a null-aware join now records how many of its leading keys
-are `NOT IN` value keys:
+join. Its first equi-join keys are the tuple elements that can be NULL. A
+nullable element cannot be NULL here when a correlated filter of the subquery
+rejects NULL for it, such as `y > o.c` for `y`. The elements that cannot be
+NULL and any equality correlations follow as further equi-join keys when their
+types can be hashed, and otherwise stay in the join filter. `Null`,
+dictionary, run-end encoded and union elements, and elements that read a
+column of these types, count as elements that can be NULL, because they can
+hold a NULL that their field's nullability does not show. Like any element
+that can be NULL, such an element must be a hashable equi-join key, so the
+planner rejects a run-end encoded one. A null-aware join therefore records how
+many of its leading keys are `NOT IN` value keys:
- `datafusion_expr::logical_plan::Join` has a new `null_aware_value_keys: usize`
field.