Skip to content

Apache DataFusion SQL Query Engine

fix: reject a multi-key null-aware join without a NOT IN value key count (#27111)

Liang-Chi Hsieh22aedf0a7155
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rs
index b98b2b97be..406fa8537e 100644
--- a/datafusion/core/tests/sql/joins.rs
+++ b/datafusion/core/tests/sql/joins.rs
@@ -774,3 +774,75 @@ async fn multi_column_not_in_sees_null_type_values() -> Result<()> {
assert_batches_eq!(["++", "++"], &filtered);
Ok(())
}
+
+/// The `NOT IN` value key count of every decorrelated mark join in `sql`'s
+/// optimized plan, with whether the join is null-aware.
+async fn mark_join_value_keys(
+ ctx: &SessionContext,
+ sql: &str,
+) -> Result<Vec<(bool, usize)>> {
+ use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
+ use datafusion::logical_expr::{JoinType, LogicalPlan};
+
+ let plan = ctx.sql(sql).await?.into_optimized_plan()?;
+ let mut joins = vec![];
+ plan.apply(|plan| {
+ if let LogicalPlan::Join(join) = plan
+ && join.join_type == JoinType::LeftMark
+ {
+ joins.push((join.null_aware, join.null_aware_value_keys));
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ Ok(joins)
+}
+
+/// Decorrelation sets the value key count only on a null-aware join, which is
+/// the only one that reads it. Any other join keeps it unset (`0`), the same as a
+/// join built through `Join::try_new` or `LogicalPlanBuilder`, so equal plans
+/// compare equal however they were built.
+#[tokio::test]
+async fn decorrelated_mark_join_sets_value_keys_only_when_null_aware() -> Result<()> {
+ let ctx = SessionContext::new();
+ for (name, nullable) in [("t", false), ("n", true)] {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, nullable),
+ Field::new("b", DataType::Int32, nullable),
+ ]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2])),
+ Arc::new(Int32Array::from(vec![3, 4])),
+ ],
+ )?;
+ ctx.register_table(
+ name,
+ Arc::new(MemTable::try_new(schema, vec![vec![batch]])?),
+ )?;
+ }
+
+ // Non-nullable keys: neither the scalar nor the multi-column `IN` is
+ // null-aware.
+ for sql in [
+ "SELECT a IN (SELECT a FROM t AS s) FROM t",
+ "SELECT (a, b) IN (SELECT a, b FROM t AS s) FROM t",
+ ] {
+ assert_eq!(
+ mark_join_value_keys(&ctx, sql).await?,
+ [(false, 0)],
+ "{sql}"
+ );
+ }
+ // Nullable keys: both are null-aware and state their count.
+ assert_eq!(
+ mark_join_value_keys(&ctx, "SELECT a IN (SELECT a FROM n AS s) FROM n").await?,
+ [(true, 1)]
+ );
+ assert_eq!(
+ mark_join_value_keys(&ctx, "SELECT (a, b) IN (SELECT a, b FROM n AS s) FROM n")
+ .await?,
+ [(true, 2)]
+ );
+ Ok(())
+}
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index c9575e7a55..ab0fef5128 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -1440,7 +1440,7 @@ impl LogicalPlanBuilder {
schema: DFSchemaRef::new(join_schema),
null_equality,
null_aware,
- null_aware_value_keys: 1,
+ null_aware_value_keys: 0,
})))
}
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index c5b5b23ba8..f09ea9cb1c 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -4652,6 +4652,11 @@ pub struct Join {
/// after them are correlation scope keys of a correlated subquery. A NULL
/// in a value key follows SQL three-valued logic across the whole tuple,
/// while the scope keys only select which subquery rows are compared.
+ ///
+ /// `0` leaves the count unset, which is the default of [`Self::try_new`]
+ /// and the `LogicalPlanBuilder` join methods. Physical planning accepts an
+ /// unset count only for a single-key join, a scalar `NOT IN`, and rejects
+ /// it for more keys: no default can tell which of them are value keys.
pub null_aware_value_keys: usize,
}
@@ -4880,7 +4885,7 @@ impl Join {
schema: Arc::new(join_schema),
null_equality,
null_aware,
- null_aware_value_keys: 1,
+ null_aware_value_keys: 0,
})
}
@@ -7230,7 +7235,7 @@ mod tests {
schema: Arc::new(left_schema.join(&right_schema)?),
null_equality: NullEquality::NullEqualsNothing,
null_aware: false,
- null_aware_value_keys: 1,
+ null_aware_value_keys: 0,
}))
}
diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
index a3d25ad4b7..85677a1f25 100644
--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
@@ -810,7 +810,11 @@ fn subquery_output_expr(subquery: &LogicalPlan, i: usize) -> Result<Expr> {
}
/// Sets [`Join::null_aware_value_keys`](datafusion_expr::Join::null_aware_value_keys)
-/// on a join built by [`build_join`].
+/// on a null-aware join built by [`build_join`]. Only a null-aware join reads
+/// the count, so any other join keeps it unset, as [`Join::try_new`] leaves
+/// it, and equal joins compare equal however they were built.
+///
+/// [`Join::try_new`]: datafusion_expr::Join::try_new
fn with_null_aware_value_keys(
plan: LogicalPlan,
null_aware_value_keys: usize,
@@ -818,6 +822,9 @@ fn with_null_aware_value_keys(
let LogicalPlan::Join(join) = plan else {
return internal_err!("expected a join, got {}", plan.display());
};
+ if !join.null_aware {
+ return Ok(LogicalPlan::Join(join));
+ }
Ok(LogicalPlan::Join(
join.with_null_aware_value_keys(null_aware_value_keys),
))
diff --git a/datafusion/optimizer/src/eliminate_cross_join.rs b/datafusion/optimizer/src/eliminate_cross_join.rs
index 9bccdfaac4..68497d2cff 100644
--- a/datafusion/optimizer/src/eliminate_cross_join.rs
+++ b/datafusion/optimizer/src/eliminate_cross_join.rs
@@ -345,7 +345,7 @@ fn find_inner_join(
schema: join_schema,
null_equality,
null_aware: false,
- null_aware_value_keys: 1,
+ null_aware_value_keys: 0,
}));
}
}
@@ -369,7 +369,7 @@ fn find_inner_join(
join_constraint: JoinConstraint::On,
null_equality,
null_aware: false,
- null_aware_value_keys: 1,
+ null_aware_value_keys: 0,
}))
}
@@ -1443,7 +1443,7 @@ mod tests {
schema: join_schema,
null_equality: NullEquality::NullEqualsNull, // Test preservation
null_aware: false,
- null_aware_value_keys: 1,
+ null_aware_value_keys: 0,
});
// Apply filter that can create join conditions
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index e22cf6dc2d..3da890acb8 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -348,7 +348,7 @@ impl HashJoinExecBuilder {
column_indices: vec![],
null_equality: NullEquality::NullEqualsNothing,
null_aware: false,
- null_aware_value_keys: 1,
+ null_aware_value_keys: 0,
dynamic_filter: None,
// Will be computed at when plan will be built.
cache: stub_properties(),
@@ -486,6 +486,10 @@ impl HashJoinExecBuilder {
filter.build_accumulator = OnceLock::new();
}
}
+ // Before `not_in`, whose value key checks assume some keys.
+ if exec.on.is_empty() {
+ return plan_err!("On constraints in HashJoinExec should be non-empty");
+ }
// Validate null_aware flag
exec.not_in()?;
@@ -518,9 +522,6 @@ impl HashJoinExecBuilder {
let left_schema = left.schema();
let right_schema = right.schema();
- if on.is_empty() {
- return plan_err!("On constraints in HashJoinExec should be non-empty");
- }
check_join_is_valid(&left_schema, &right_schema, &on)?;
let (join_schema, column_indices) =
@@ -864,6 +865,11 @@ pub struct HashJoinExec {
/// Such a join decides UNKNOWN per row, like a correlated scalar `NOT IN`:
/// without scope keys every NULL-valued row is paired with every row on
/// the other side to check its other elements.
+ ///
+ /// `0` leaves the count unset, the default of [`HashJoinExecBuilder::new`]
+ /// and [`Self::try_new`]. An unset count is accepted only for a single-key
+ /// join, a scalar `NOT IN`; a null-aware join with more keys must set it
+ /// with [`HashJoinExecBuilder::with_null_aware_value_keys`].
pub null_aware_value_keys: usize,
/// Cache holding plan properties like equivalences, output partitioning etc.
cache: Arc<PlanProperties>,
@@ -1425,7 +1431,7 @@ impl DisplayAs for HashJoinExec {
let display_null_aware =
match (self.null_aware, self.null_aware_value_keys) {
(false, _) => String::new(),
- (true, 1) => ", null_aware".to_string(),
+ (true, 0 | 1) => ", null_aware".to_string(),
(true, n) => format!(", null_aware(value_keys={n})"),
};
let display_prepared = self
@@ -1479,7 +1485,7 @@ impl DisplayAs for HashJoinExec {
}
if self.null_aware {
- if self.null_aware_value_keys == 1 {
+ if self.null_aware_value_keys <= 1 {
writeln!(f, "null_aware")?;
} else {
writeln!(
@@ -2290,16 +2296,14 @@ impl HashJoinExec {
.with_partition_mode(partition_mode)
.with_null_equality(null_equality)
.with_null_aware(*null_aware)
- // Messages predating the field decode it as 0; they can only hold
- // scalar `NOT IN` joins.
- .with_null_aware_value_keys(
- usize_from_wire(
- u64::from(*null_aware_value_keys),
- "HashJoinExec",
- "null_aware_value_keys",
- )?
- .max(1),
- )
+ // `0` is an unset count, which `build` accepts only for a single
+ // key. Messages predating the field decode it as 0; they can only
+ // hold scalar `NOT IN` joins.
+ .with_null_aware_value_keys(usize_from_wire(
+ u64::from(*null_aware_value_keys),
+ "HashJoinExec",
+ "null_aware_value_keys",
+ )?)
.with_fetch(fetch)
.build()?;
@@ -2595,8 +2599,9 @@ mod proto_tests {
}
/// A multi-column `NOT IN` keeps its value key count through the round
- /// trip; a node written before the field existed decodes it as 0, which
- /// is read as a scalar `NOT IN`.
+ /// trip. A count of 0, which is also how a node written before the field
+ /// existed decodes it, stays unset: accepted for a single key, rejected
+ /// for more.
#[test]
fn null_aware_value_keys_round_trip() {
let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = vec![
@@ -2624,7 +2629,17 @@ mod proto_tests {
assert_eq!(as_join(&decode(node.clone())).null_aware_value_keys, 2);
node.null_aware_value_keys = 0;
- assert_eq!(as_join(&decode(node)).null_aware_value_keys, 1);
+ let decoder = StubPlanDecoder::ok();
+ let ctx = ExecutionPlanDecodeCtx::new(&decoder);
+ let err =
+ HashJoinExec::try_from_proto(&join_node(node.clone()), &ctx).unwrap_err();
+ assert!(
+ err.to_string().contains("with_null_aware_value_keys"),
+ "{err}"
+ );
+
+ node.on.truncate(1);
+ assert_eq!(as_join(&decode(node)).null_aware_value_keys, 0);
}
#[test]
@@ -10292,6 +10307,25 @@ mod tests {
);
}
+ /// A null-aware join for a scalar `NOT IN` on `on[0]`, correlated by any
+ /// keys after it: it states one `NOT IN` value key, which a multi-key
+ /// join must.
+ fn try_new_scalar_not_in(
+ left: Arc<dyn ExecutionPlan>,
+ right: Arc<dyn ExecutionPlan>,
+ on: JoinOn,
+ filter: Option<JoinFilter>,
+ join_type: JoinType,
+ partition_mode: PartitionMode,
+ ) -> Result<HashJoinExec> {
+ HashJoinExecBuilder::new(left, right, on, join_type)
+ .with_filter(filter)
+ .with_partition_mode(partition_mode)
+ .with_null_aware(true)
+ .with_null_aware_value_keys(1)
+ .build()
+ }
+
/// Test that null_aware validation rejects PartitionMode::Partitioned (must be CollectLeft).
#[tokio::test]
async fn test_null_aware_validation_partitioned() {
@@ -10310,16 +10344,13 @@ mod tests {
];
// A correlated (multi-key) LeftMark requires CollectLeft.
- let result = HashJoinExec::try_new(
+ let result = try_new_scalar_not_in(
left,
right,
on,
None,
- &JoinType::LeftMark,
- None,
+ JoinType::LeftMark,
PartitionMode::Partitioned,
- NullEquality::NullEqualsNothing,
- true, // null_aware = true (invalid with Partitioned)
);
assert!(result.is_err());
@@ -10357,16 +10388,13 @@ mod tests {
if multi_key {
on.push(key("dummy", "dummy")?);
}
- let join = HashJoinExec::try_new(
+ let join = try_new_scalar_not_in(
Arc::clone(&left),
Arc::clone(&right),
on,
with_filter.then(prepare_join_filter),
- &join_type,
- None,
+ join_type,
PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- true,
)?;
let scope = NotInScope::Correlated { value_keys: 1 };
let expected = match join_type {
@@ -10680,16 +10708,13 @@ mod tests {
),
];
- let join = HashJoinExec::try_new(
+ let join = try_new_scalar_not_in(
left,
right,
on,
None,
- &JoinType::LeftMark,
- None,
+ JoinType::LeftMark,
PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- true,
)?;
let stream = join.execute(0, task_ctx)?;
@@ -10748,16 +10773,13 @@ mod tests {
),
];
- let join = HashJoinExec::try_new(
+ let join = try_new_scalar_not_in(
left,
right,
on,
None,
- &JoinType::LeftMark,
- None,
+ JoinType::LeftMark,
PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- true,
)?;
let stream = join.execute(0, task_ctx)?;
@@ -10813,16 +10835,13 @@ mod tests {
Arc::new(Column::new_with_schema("grp", &right.schema())?) as _,
),
];
- let join = HashJoinExec::try_new(
+ let join = try_new_scalar_not_in(
left,
right,
on,
None,
- &JoinType::LeftAnti,
- None,
+ JoinType::LeftAnti,
PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- true,
)?;
let task_ctx = fallback_context(memory_limit);
let batches = common::collect(join.execute(0, task_ctx)?).await?;
@@ -11032,35 +11051,103 @@ mod tests {
Ok(())
}
- /// The value key count must name at least one key and no more than the
- /// join has.
- #[tokio::test]
- async fn test_null_aware_validation_value_keys() -> Result<()> {
- for value_keys in [0, 3] {
- let left = build_table_two_cols(("a", &vec![Some(1)]), ("b", &vec![Some(2)]));
- let right =
- build_table_two_cols(("x", &vec![Some(1)]), ("y", &vec![Some(2)]));
- let on = vec![
- (
- Arc::new(Column::new_with_schema("a", &left.schema())?) as _,
- Arc::new(Column::new_with_schema("x", &right.schema())?) as _,
- ),
- (
- Arc::new(Column::new_with_schema("b", &left.schema())?) as _,
- Arc::new(Column::new_with_schema("y", &right.schema())?) as _,
- ),
- ];
- let err = HashJoinExecBuilder::new(left, right, on, JoinType::LeftAnti)
- .with_partition_mode(PartitionMode::CollectLeft)
+ /// The value key count must name no more keys than the join has. An
+ /// unset count (`0`) is a scalar `NOT IN` with a single key, and is
+ /// rejected with more keys rather than guessing which of them are `NOT IN`
+ /// value keys and which are correlation scope keys.
+ #[test]
+ fn test_null_aware_validation_value_keys() -> Result<()> {
+ let left = build_table_two_cols(("a", &vec![Some(1)]), ("b", &vec![None]));
+ let right = build_table_two_cols(("x", &vec![Some(1)]), ("y", &vec![Some(5)]));
+ let key = |l: &str, r: &str| -> Result<(PhysicalExprRef, PhysicalExprRef)> {
+ Ok((
+ Arc::new(Column::new_with_schema(l, &left.schema())?) as _,
+ Arc::new(Column::new_with_schema(r, &right.schema())?) as _,
+ ))
+ };
+ let two_keys = vec![key("a", "x")?, key("b", "y")?];
+ let build = |on: JoinOn, join_type, partition_mode, value_keys| {
+ HashJoinExecBuilder::new(Arc::clone(&left), Arc::clone(&right), on, join_type)
+ .with_partition_mode(partition_mode)
.with_null_aware(true)
.with_null_aware_value_keys(value_keys)
.build()
- .unwrap_err();
- assert!(
- err.to_string()
- .contains("needs between 1 and 2 `NOT IN` value keys"),
- "{err}"
- );
+ };
+
+ for (value_keys, expected) in [
+ (
+ 0,
+ "join with 2 keys needs the number of its `NOT IN` value keys, set it with `with_null_aware_value_keys`",
+ ),
+ (3, "needs between 1 and 2 `NOT IN` value keys"),
+ ] {
+ let err = build(
+ two_keys.clone(),
+ JoinType::LeftAnti,
+ PartitionMode::CollectLeft,
+ value_keys,
+ )
+ .unwrap_err();
+ assert!(err.to_string().contains(expected), "{err}");
+ }
+
+ // `HashJoinExec::try_new` cannot set the count, so it only builds
+ // single-key null-aware joins, as before value keys existed.
+ let err = HashJoinExec::try_new(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ two_keys.clone(),
+ None,
+ &JoinType::LeftAnti,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ true,
+ )
+ .unwrap_err();
+ assert!(
+ err.to_string().contains("with_null_aware_value_keys"),
+ "{err}"
+ );
+
+ let join = build(
+ vec![key("a", "x")?],
+ JoinType::LeftAnti,
+ PartitionMode::CollectLeft,
+ 0,
+ )?;
+ assert_eq!(join.not_in()?.map(NotIn::value_keys), Some(1));
+
+ // A join that no count can make valid reports what is wrong with it,
+ // not the missing count.
+ for (on, join_type, partition_mode, expected) in [
+ (
+ two_keys.clone(),
+ JoinType::Inner,
+ PartitionMode::CollectLeft,
+ "null_aware can only be true for LeftAnti joins",
+ ),
+ (
+ two_keys.clone(),
+ JoinType::RightAnti,
+ PartitionMode::Partitioned,
+ "null_aware can only be true for LeftAnti joins",
+ ),
+ (
+ two_keys,
+ JoinType::LeftAnti,
+ PartitionMode::Partitioned,
+ "null_aware joins require PartitionMode::CollectLeft",
+ ),
+ (
+ vec![],
+ JoinType::LeftAnti,
+ PartitionMode::CollectLeft,
+ "On constraints in HashJoinExec should be non-empty",
+ ),
+ ] {
+ let err = build(on, join_type, partition_mode, 0).unwrap_err();
+ assert!(err.to_string().contains(expected), "{join_type}: {err}");
}
Ok(())
}
@@ -11179,16 +11266,13 @@ mod tests {
// The outer side `left` as the build side (`LeftAnti`), and swapped to
// the probe side (`RightAnti`), must keep the same rows.
- let left_anti = HashJoinExec::try_new(
+ let left_anti = try_new_scalar_not_in(
Arc::clone(&left),
Arc::clone(&right),
on.clone(),
None,
- &JoinType::LeftAnti,
- None,
+ JoinType::LeftAnti,
PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- true,
)?;
let right_anti = swap_for_separate_execution(&left_anti)?;
let batches =
@@ -11332,16 +11416,13 @@ mod tests {
))
})
.collect::<Result<JoinOn>>()?;
- let join = HashJoinExec::try_new(
+ let join = try_new_scalar_not_in(
left,
right,
on,
None,
- &join_type,
- None,
+ join_type,
PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- true,
)?;
let batches = common::collect(join.execute(0, task_ctx)?).await?;
results.push(batches_to_sort_string(&batches));
@@ -11425,16 +11506,13 @@ mod tests {
})
.collect::<Result<JoinOn>>()?;
- let join = HashJoinExec::try_new(
+ let join = try_new_scalar_not_in(
left,
right,
on,
None,
- &JoinType::LeftAnti,
- None,
+ JoinType::LeftAnti,
PartitionMode::CollectLeft,
- NullEquality::NullEqualsNothing,
- true,
)?;
let stream = join.execute(0, task_ctx)?;
diff --git a/datafusion/physical-plan/src/joins/hash_join/not_in.rs b/datafusion/physical-plan/src/joins/hash_join/not_in.rs
index 46c835bd42..3e0574a5a7 100644
--- a/datafusion/physical-plan/src/joins/hash_join/not_in.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/not_in.rs
@@ -102,30 +102,14 @@ impl NotIn {
num_value_keys: usize,
has_filter: bool,
) -> Result<Self> {
- if num_value_keys == 0 || num_value_keys > num_keys {
- return plan_err!(
- "null_aware {join_type} join needs between 1 and {num_keys} `NOT IN` value keys, got {num_value_keys}"
- );
- }
- // `num_keys > 1` also covers a multi-column value key without any
- // correlation: a NULL in one tuple element leaves the comparison FALSE
- // whenever another element is a definite mismatch, so it too must be
- // decided per row.
- let scope = if num_keys > 1 || has_filter {
- NotInScope::Correlated {
- value_keys: num_value_keys,
- }
- } else {
- NotInScope::Uncorrelated
- };
+ // The join type and mode come first: no value key count can make up
+ // for them. `None` is `RightAnti`, which emits probe rows.
let output = match (join_type, partition_mode) {
- (JoinType::LeftAnti, _) => NotInOutput::Rows,
- (JoinType::LeftMark, _) => NotInOutput::Mark,
+ (JoinType::LeftAnti, _) => Some(NotInOutput::Rows),
+ (JoinType::LeftMark, _) => Some(NotInOutput::Mark),
// `PartitionMode::CollectLeft` is safe because `RightAnti` is
// probe-driven: every probe row sees the whole build side.
- (JoinType::RightAnti, PartitionMode::CollectLeft) => {
- return Ok(Self::ProbeRows { scope });
- }
+ (JoinType::RightAnti, PartitionMode::CollectLeft) => None,
_ => {
return plan_err!(
"null_aware can only be true for LeftAnti joins and RightAnti joins with `CollectLeft` `PartitionMode`, or LeftMark joins, got {join_type} with {partition_mode}"
@@ -142,7 +126,39 @@ impl NotIn {
"null_aware joins require PartitionMode::CollectLeft, got PartitionMode::Partitioned"
);
}
- Ok(Self::BuildRows { scope, output })
+ // `0` is an unset count, which is only unambiguous with a single key:
+ // a scalar `NOT IN`. With more keys, any default would silently pick
+ // which of them are `NOT IN` value keys and which are correlation
+ // scope keys.
+ let num_value_keys = match num_value_keys {
+ 0 if num_keys == 1 => 1,
+ 0 => {
+ return plan_err!(
+ "null_aware {join_type} join with {num_keys} keys needs the number of its `NOT IN` value keys, set it with `with_null_aware_value_keys`"
+ );
+ }
+ n if n > num_keys => {
+ return plan_err!(
+ "null_aware {join_type} join needs between 1 and {num_keys} `NOT IN` value keys, got {n}"
+ );
+ }
+ n => n,
+ };
+ // `num_keys > 1` also covers a multi-column value key without any
+ // correlation: a NULL in one tuple element leaves the comparison FALSE
+ // whenever another element is a definite mismatch, so it too must be
+ // decided per row.
+ let scope = if num_keys > 1 || has_filter {
+ NotInScope::Correlated {
+ value_keys: num_value_keys,
+ }
+ } else {
+ NotInScope::Uncorrelated
+ };
+ Ok(match output {
+ Some(output) => Self::BuildRows { scope, output },
+ None => Self::ProbeRows { scope },
+ })
}
fn scope(self) -> NotInScope {
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index 341b2dbd3d..ee1d803bd2 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -275,8 +275,9 @@ message JoinNode {
LogicalExprNode filter = 8;
bool null_aware = 9;
// Number of `NOT IN` value keys of a null-aware join (see
- // `Join::null_aware_value_keys`). Messages predating this field decode it
- // as 0, which is read as 1: they can only hold scalar `NOT IN` joins.
+ // `Join::null_aware_value_keys`). 0 is an unset count, accepted only for a
+ // single key. Messages predating this field decode it as 0: they can only
+ // hold scalar `NOT IN` joins.
uint32 null_aware_value_keys = 10;
}
@@ -1465,9 +1466,9 @@ message HashJoinExecNode {
// `None`, which is the correct reading of an older message.
optional uint64 fetch = 12;
// Number of `NOT IN` value keys of a null-aware join (see
- // `HashJoinExec::null_aware_value_keys`). Messages predating this field
- // decode it as 0, which is read as 1: they can only hold scalar `NOT IN`
- // joins.
+ // `HashJoinExec::null_aware_value_keys`). 0 is an unset count, accepted
+ // only for a single key. Messages predating this field decode it as 0:
+ // they can only hold scalar `NOT IN` joins.
uint32 null_aware_value_keys = 13;
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index 142e560bae..06342881ec 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -442,8 +442,9 @@ pub struct JoinNode {
#[prost(bool, tag = "9")]
pub null_aware: bool,
/// Number of `NOT IN` value keys of a null-aware join (see
- /// `Join::null_aware_value_keys`). Messages predating this field decode it
- /// as 0, which is read as 1: they can only hold scalar `NOT IN` joins.
+ /// `Join::null_aware_value_keys`). 0 is an unset count, accepted only for a
+ /// single key. Messages predating this field decode it as 0: they can only
+ /// hold scalar `NOT IN` joins.
#[prost(uint32, tag = "10")]
pub null_aware_value_keys: u32,
}
@@ -2235,9 +2236,9 @@ pub struct HashJoinExecNode {
#[prost(uint64, optional, tag = "12")]
pub fetch: ::core::option::Option<u64>,
/// Number of `NOT IN` value keys of a null-aware join (see
- /// `HashJoinExec::null_aware_value_keys`). Messages predating this field
- /// decode it as 0, which is read as 1: they can only hold scalar `NOT IN`
- /// joins.
+ /// `HashJoinExec::null_aware_value_keys`). 0 is an unset count, accepted
+ /// only for a single key. Messages predating this field decode it as 0:
+ /// they can only hold scalar `NOT IN` joins.
#[prost(uint32, tag = "13")]
pub null_aware_value_keys: u32,
}
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index 592d56cb44..bd4f100c28 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -1068,16 +1068,15 @@ impl AsLogicalPlan for LogicalPlanNode {
NullEquality::from(null_equality),
join.null_aware,
)?
- // Plans encoded before the field existed decode it as 0; they
- // could only hold scalar `NOT IN` joins.
- .with_null_aware_value_keys(
- usize_from_wire(
- join.null_aware_value_keys,
- "JoinNode",
- "null_aware_value_keys",
- )?
- .max(1),
- ),
+ // `0` is an unset count, which physical planning accepts
+ // only for a single key. Plans encoded before the field
+ // existed decode it as 0; they could only hold scalar
+ // `NOT IN` joins.
+ .with_null_aware_value_keys(usize_from_wire(
+ join.null_aware_value_keys,
+ "JoinNode",
+ "null_aware_value_keys",
+ )?),
))
}
LogicalPlanType::AsOfJoin(join) => {
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index bd7f5edc92..849f3237d8 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1063,42 +1063,28 @@ Wire compatibility is directional:
### Null-aware joins record their `NOT IN` value key count
Multi-column `(a, b) NOT IN (SELECT x, y ...)` is now planned as a null-aware
-join. Its first equi-join keys are the tuple elements that can be NULL. A
-nullable element cannot be NULL here when a correlated filter of the subquery
-rejects NULL for it, such as `y > o.c` for `y`. The elements that cannot be
-NULL and any equality correlations follow as further equi-join keys when their
-types can be hashed, and otherwise stay in the join filter. `Null`,
-dictionary, run-end encoded and union elements, and elements that read a
-column of these types, count as elements that can be NULL, because they can
-hold a NULL that their field's nullability does not show. Like any element
-that can be NULL, such an element must be a hashable equi-join key, so the
-planner rejects a run-end encoded one. A null-aware join therefore records how
-many of its leading keys are `NOT IN` value keys:
-
-- `datafusion_expr::logical_plan::Join` has a new `null_aware_value_keys: usize`
- field.
-- `HashJoinExec` has a new `null_aware_value_keys: usize` field, set with
- `HashJoinExecBuilder::with_null_aware_value_keys`.
-
-The count is only read when `null_aware` is set. `Join::try_new`, the
-`LogicalPlanBuilder` join methods and `HashJoinExec::try_new` set it to `1`,
-the scalar `NOT IN` of earlier releases. Code constructing `Join` with a struct
-literal must initialize the field:
+join, which records how many of its leading equi-join keys are `NOT IN` value
+keys; the keys after them are correlation scope keys. `Join` and
+`HashJoinExec` have a new `null_aware_value_keys: usize` field, set with
+`with_null_aware_value_keys`, and so do the generated protobuf `JoinNode` and
+`HashJoinExecNode` types.
+
+The count defaults to `0`, unset. An unset count is accepted only for a
+single-key null-aware join, the scalar `NOT IN` of earlier releases, so
+existing code and 55.x plans keep working. Code that builds a multi-key
+null-aware join itself must set the count, or the join fails to plan:
```rust,ignore
-let join = Join {
- // ...
- null_aware,
- null_aware_value_keys: 1,
-};
+let join = HashJoinExecBuilder::new(left, right, on, JoinType::LeftAnti)
+ .with_partition_mode(PartitionMode::CollectLeft)
+ .with_null_aware(true)
+ // `on[..2]` are the `NOT IN` value keys, any later keys scope keys.
+ .with_null_aware_value_keys(2)
+ .build()?;
```
-The generated protobuf `JoinNode` and `HashJoinExecNode` types have a new
-`null_aware_value_keys` field, which struct literals must provide too. A 56.0
-reader decodes a payload without the field (value `0`) as a scalar `NOT IN`. A
-55.0 reader must not consume a 56.0 plan holding a multi-column `NOT IN`: it
-ignores the unknown field and would read the extra tuple elements as
-correlation keys, returning wrong results.
+A 55.x reader ignores the field and rejects a multi-key null-aware join from a
+56.0 plan.
### Null-aware sort-merge joins add a protobuf field