Apache DataFusion SQL Query Engine
fix: reject probe rows routed to a canceled hash join partition (#27092)
Adrian Garcia Badaracco954552d659ac
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex f1d393e6ff..6f794b6b00 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -10542,8 +10542,8 @@ mod tests {/// Here the build-side logical NULL lives in exactly one build partition, so/// the `keys_have_null` flag has to survive the per-partition aggregation in/// `build_partitioned_filter` and widen the *routed* filter as a whole: the- /// probe NULL is routed to a single `CASE` branch, and OR-ing `IS NULL` into- /// that one branch would still drop it for every other route.+ /// probe NULL is routed to a single partition, and OR-ing `IS NULL` into+ /// that one partition's check would still drop it for every other route.#[tokio::test]async fn test_partitioned_null_equal_dynamic_filter_keeps_probe_nulls_for_build_logical_null()-> Result<()> {diff --git a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rsindex ffe8046c57..eabc1bc86f 100644--- a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs+++ b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs@@ -59,9 +59,8 @@ impl SeededRandomState {/// This expression computes the hash of join key columns using a specific RandomState./// It returns a UInt64Array containing the hash values.///-/// This is used for:-/// - Computing routing hashes (with RepartitionExec's 0,0,0,0 seeds)-/// - Computing lookup hashes (with HashJoin's 'J','O','I','N' seeds)+/// The hash join does not build this expression. It exists so that+/// `datafusion-proto` can still decode serialized plans that contain it.pub struct HashExpr {/// Columns to hashon_columns: Vec<PhysicalExprRef>,@@ -354,8 +353,9 @@ impl HashTableLookupExpr {/// * `pruning_bitmap` - key-range summary covering every partition's keys/// for pruning only, or `None`///- /// A single non-empty map is probed directly without routing, so it may be- /// any [`Map`] and `router` is ignored. Otherwise `router` is required.+ /// When `maps` has exactly one entry, that map is probed directly without+ /// routing and `router` is ignored. Otherwise `router` is required, even if+ /// only one entry is non-empty.////// # Public Only for Internal Use:/// `datafusion-proto` tests require this constructor, but it is not part ofdiff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rsindex 0a3ce6dcd3..091cc3a34f 100644--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs@@ -18,7 +18,7 @@ use crate::joins::hash_join::exec::HASH_JOIN_SEED;use crate::joins::hash_join::inlist_builder::build_struct_fields;use crate::joins::hash_join::not_in::NotIn;use crate::joins::hash_join::partitioned_hash_eval::{- HashExpr, HashTableLookupExpr, PartitionRouter, SeededRandomState,+ HashTableLookupExpr, PartitionRouter, SeededRandomState,};use crate::joins::join_hash_map::{JoinHashMapType, JoinHashMapU32};use crate::joins::key_range_bitmap::{KeyRangeBitmap, KeyRangePart};@@ -35,7 +35,7 @@ use datafusion_common::{use datafusion_expr::Operator;use datafusion_functions::core::r#struct as struct_func;use datafusion_physical_expr::expressions::{- BinaryExpr, CaseExpr, DynamicFilterPhysicalExpr, InListExpr, IsNullExpr, lit,+ BinaryExpr, DynamicFilterPhysicalExpr, InListExpr, IsNullExpr, lit,};use datafusion_physical_expr::{PhysicalExpr, PhysicalExprRef, RangePartitioning, ScalarFunctionExpr,@@ -222,7 +222,7 @@ fn create_bounds_predicate(/// Combines a membership predicate and a bounds predicate with logical AND.////// Returns `None` when neither is available; callers decide the fallback (e.g.-/// skip updating the filter vs. emit a `lit(true)` branch inside a CASE).+/// skip updating the filter or use `lit(true)`).fn combine_membership_and_bounds(membership_expr: Option<Arc<dyn PhysicalExpr>>,bounds_expr: Option<Arc<dyn PhysicalExpr>>,@@ -495,7 +495,7 @@ impl SharedBuildAccumulator {/// This unified method handles both CollectLeft and Partitioned modes. When all partitions/// have reported (barrier wait), the leader builds the appropriate filter expression:/// - CollectLeft: Simple conjunction of bounds and membership check- /// - Partitioned: CASE expression routing to per-partition filters+ /// - Partitioned: one lookup that routes each probe row to its partition's keys////// # Arguments/// * `data` - Build data including hash map, pushdown strategy, and bounds@@ -708,167 +708,62 @@ impl SharedBuildAccumulator {}}- /// Builds one routed probe-side filter from finalized partitioned build data.- /// Empty partitions reject their routed rows, while canceled partitions stay- /// permissive because their build contents are unknown.+ /// Builds one probe-side filter from finalized partitioned build data.+ /// Empty and canceled partitions contribute no key.fn build_partitioned_filter(&self, partitions: Vec<PartitionStatus>) -> Result<()> {- let mut partition_filters = Vec::with_capacity(partitions.len());- let mut real_partition_ids = Vec::new();- let mut empty_partition_ids = Vec::new();- let mut has_canceled_unknown = false;- let mut keys_have_null = false;- // Every non-empty partition's membership strategy and bounds, merged into- // one membership filter below; `None` once a partition cannot be merged.- let mut members = Some(Vec::with_capacity(partitions.len()));+ let num_partitions = partitions.len();+ let mut members = Vec::with_capacity(num_partitions);let mut merged_bounds: Option<PartitionBounds> = None;+ let mut keys_have_null = false;for (partition_id, partition) in partitions.into_iter().enumerate() {match partition {- PartitionStatus::Reported(PartitionData {+ // A canceled partition's stream never probes, so no key of its+ // build side can matter.+ PartitionStatus::CanceledUnknown+ | PartitionStatus::Reported(PartitionData {pushdown: PushdownStrategy::Empty,..- }) => {- empty_partition_ids.push(partition_id);- partition_filters.push(lit(false));- }+ }) => {}PartitionStatus::Reported(PartitionData {bounds,pushdown,keys_have_null: partition_keys_have_null,}) => {- real_partition_ids.push(partition_id);keys_have_null |= partition_keys_have_null;- members = members.map(|mut members| {- members.push((partition_id, pushdown.clone(), bounds.clone()));- members- });merged_bounds = Some(match merged_bounds {None => bounds.clone(),Some(merged) => merged.union(&bounds),});-- let membership_expr = create_membership_predicate(- &self.on_right,- pushdown,- &HASH_JOIN_SEED,- self.probe_schema.as_ref(),- )?;- let bounds_expr = create_bounds_predicate(&self.on_right, &bounds);- let then_expr =- combine_membership_and_bounds(membership_expr, bounds_expr)- .unwrap_or_else(|| lit(true));- partition_filters.push(then_expr);- }- PartitionStatus::CanceledUnknown => {- has_canceled_unknown = true;- partition_filters.push(lit(true));- // Rows routed here must pass, which a merged filter cannot express.- members = None;- // A canceled partition's build content is unknown, so it- // may hold a NULL key.- keys_have_null = true;+ members.push((partition_id, pushdown, bounds));}PartitionStatus::Pending => {- return datafusion_common::internal_err!(+ return internal_err!("attempted to finalize dynamic filter with pending partition");}}}- let all_partitions_canceled = has_canceled_unknown- && real_partition_ids.is_empty()- && empty_partition_ids.is_empty();- let all_partitions_empty = !has_canceled_unknown && real_partition_ids.is_empty();- let one_non_empty_partition =- !has_canceled_unknown && real_partition_ids.len() == 1;-- let filter_expr = if all_partitions_canceled {- // No build data is known, so filtering any probe row could discard a match.- lit(true)- } else if all_partitions_empty {+ let filter_expr = match members.as_slice() {// No build row exists, so no probe row can match.- lit(false)- } else if one_non_empty_partition {- // Only one build partition contains rows, so its filter covers every- // possible probe match without routing.- Arc::clone(&partition_filters[real_partition_ids[0]])- } else if let Some(members) = members {- self.merged_membership_filter(- &members,- merged_bounds,- partition_filters.len(),- )?- } else {- // Builds the shared sparse `CASE` for partition filter routing.- // Without cancellation, omitted branches are known empty and safely fall- // through to `ELSE false`. With cancellation, omitted canceled partitions- // have unknown contents and must fall through to `ELSE true`, so known-empty- // partitions are emitted explicitly as false branches.- let mut branches = if has_canceled_unknown {- empty_partition_ids- .iter()- .map(|&partition_id| (lit(partition_id as u64), lit(false)))- .collect::<Vec<_>>()- } else {- vec![]- };- branches.extend(real_partition_ids.iter().map(|&partition_id| {- (- lit(partition_id as u64),- Arc::clone(&partition_filters[partition_id]),- )- }));-- let routing_expr = if let Some(range_partitioning) =- &self.probe_range_partitioning- {- // Routes probe rows with the partition id selected by [`RangeExpr`].- // CASE range_partition(keys)- // WHEN empty_partition_id THEN false -- only when cancellation exists- // WHEN real_partition_id THEN F(real_partition_id)- // ...- // ELSE has_canceled_unknown- // END- assert_or_internal_err!(- partition_filters.len() == range_partitioning.partition_count(),- "Dynamic filter partition count {} does not match Range partition count {}",- partition_filters.len(),- range_partitioning.partition_count()- );- Arc::new(RangeExpr::try_new_with_schema(- self.on_right.clone(),- range_partitioning,- &self.probe_schema,- )?) as Arc<dyn PhysicalExpr>- } else {- // Routes probe rows with the same `hash(keys) % partition_count` expression used- // by Hash repartitioning.- // CASE hash(keys) % partition_count- // WHEN empty_partition_id THEN false -- only when cancellation exists- // WHEN real_partition_id THEN F(real_partition_id)- // ...- // ELSE has_canceled_unknown- // END- let routing_hash_expr = Arc::new(HashExpr::new(- self.on_right.clone(),- self.repartition_random_state.clone(),- "hash_repartition".to_string(),- )) as Arc<dyn PhysicalExpr>;- Arc::new(BinaryExpr::new(- routing_hash_expr,- Operator::Modulo,- lit(partition_filters.len() as u64),- )) as Arc<dyn PhysicalExpr>- };-- Arc::new(CaseExpr::try_new(- Some(routing_expr),- branches,- Some(lit(has_canceled_unknown)),- )?)+ [] => lit(false),+ // Only one build partition holds rows, so its own filter covers every+ // probe match without routing, and an IN list keeps its pruning.+ [(_, pushdown, bounds)] => combine_membership_and_bounds(+ create_membership_predicate(+ &self.on_right,+ pushdown.clone(),+ &HASH_JOIN_SEED,+ self.probe_schema.as_ref(),+ )?,+ create_bounds_predicate(&self.on_right, bounds),+ )+ .expect("membership predicate is always present"),+ _ => {+ self.merged_membership_filter(&members, merged_bounds, num_partitions)?+ }};-self.dynamic_filter.update(self.preserve_probe_nulls(filter_expr, keys_have_null)?)}@@ -1129,13 +1024,12 @@ mod tests {use crate::joins::hash_join::not_in::{NotInOutput, NotInScope};use crate::joins::join_hash_map::JoinHashMapU32;- use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array};- use arrow::compute::SortOptions;- use arrow::record_batch::RecordBatch;- use datafusion_common::SplitPoint;- use datafusion_physical_expr::{- PhysicalSortExpr,- expressions::{Column, Literal},+ use arrow::array::{ArrayRef, Int32Array};+ use datafusion_physical_expr::expressions::{Column, Literal};+ #[cfg(not(feature = "force_hash_collisions"))]+ use {+ arrow::array::BooleanArray, arrow::record_batch::RecordBatch,+ datafusion_common::SplitPoint, datafusion_physical_expr::PhysicalSortExpr,};fn test_on_right() -> Vec<PhysicalExprRef> {@@ -1271,11 +1165,6 @@ mod tests {.expect("expected BinaryExpr dynamic filter")}- fn case_expr(expr: &PhysicalExprRef) -> &CaseExpr {- expr.downcast_ref::<CaseExpr>()- .expect("expected CaseExpr dynamic filter")- }-fn assert_literal_bool(expr: &PhysicalExprRef, expected: bool) {let literal = expr.downcast_ref::<Literal>()@@ -1353,7 +1242,7 @@ mod tests {}#[test]- fn partitioned_hash_map_partitions_route_without_case() {+ fn partitioned_hash_map_partitions_merge_into_one_lookup() {let acc = make_partitioned_expr_accumulator_for_test(3);acc.build_filter(FinalizeInput::Partitioned(vec![@@ -1364,7 +1253,6 @@ mod tests {.unwrap();let expr = current_expr(&acc);- assert!(expr.downcast_ref::<CaseExpr>().is_none());assert_eq!(expr.to_string(),"probe_key@0 >= 1 AND probe_key@0 <= 10 AND hash_lookup"@@ -1420,7 +1308,7 @@ mod tests {}/// Partitions that chose an IN list and partitions that chose a hash table- /// merge into one lookup without a `CASE`: a probe row passes exactly when+ /// merge into one lookup: a probe row passes exactly when/// the partition its key routes to holds the key.#[test]#[cfg(not(feature = "force_hash_collisions"))]@@ -1442,11 +1330,6 @@ mod tests {))).unwrap();- assert!(- current_expr(&acc).downcast_ref::<CaseExpr>().is_none(),- "expected a merged lookup, got: {}",- current_expr(&acc)- );let probe: Vec<i32> = (0..40).collect();let expected: Vec<bool> = probe.iter().map(|key| key % 2 == 0).collect();assert_matches(&acc, &probe, &expected);@@ -1472,8 +1355,7 @@ mod tests {let expr = current_expr(&acc);assert!(- expr.downcast_ref::<CaseExpr>().is_none()- && expr.downcast_ref::<InListExpr>().is_none(),+ expr.downcast_ref::<InListExpr>().is_none(),"expected a hash lookup, got: {expr}");let probe: Vec<i32> = (0..20).collect();@@ -1533,11 +1415,6 @@ mod tests {))).unwrap();- assert!(- current_expr(&acc).downcast_ref::<CaseExpr>().is_none(),- "expected a merged lookup, got: {}",- current_expr(&acc)- );let probe: Vec<i32> = (0..40).collect();let expected: Vec<bool> = probe.iter().map(|key| key % 2 == 0).collect();assert_matches(&acc, &probe, &expected);@@ -1568,7 +1445,6 @@ mod tests {reported(array_map_of(&[37, 38, 39]), bounds(37, 39)),]))?;- assert!(current_expr(&acc).downcast_ref::<CaseExpr>().is_none());let probe: Vec<i32> = (0..50).collect();let expected: Vec<bool> = probe.iter()@@ -1596,7 +1472,7 @@ mod tests {}#[test]- fn partitioned_one_real_partition_with_rest_empty_skips_case() {+ fn partitioned_one_real_partition_with_rest_empty_keeps_its_in_list() {let acc = make_partitioned_expr_accumulator_for_test(3);acc.build_filter(FinalizeInput::Partitioned(vec![@@ -1608,11 +1484,62 @@ mod tests {let expr = current_expr(&acc);in_list_expr(&expr);- assert!(expr.downcast_ref::<CaseExpr>().is_none());}+ /// One non-empty partition keeps its own filter: no routing, and an IN list+ /// stays an IN list even when its duplicate rows exceed the merged list limit.#[test]- fn partitioned_canceled_unknown_partitions_keep_unknown_routes_permissive() {+ fn partitioned_one_non_empty_partition_keeps_its_own_filter() {+ let empty = || reported(PushdownStrategy::Empty, no_bounds());+ let mut acc = make_partitioned_expr_accumulator_for_test(3);+ acc.in_list_max_values = 150;+ acc.build_filter(FinalizeInput::Partitioned(vec![+ empty(),+ reported(in_list(&[2; 1000]), no_bounds()),+ PartitionStatus::CanceledUnknown,+ ]))+ .unwrap();+ in_list_expr(¤t_expr(&acc));++ let acc = make_partitioned_expr_accumulator_for_test(3);+ acc.build_filter(FinalizeInput::Partitioned(vec![+ empty(),+ reported(hash_map(), bounds(1, 7)),+ PartitionStatus::CanceledUnknown,+ ]))+ .unwrap();+ let expr = format!("{:?}", current_expr(&acc));+ assert!(!expr.contains("routed by"), "{expr}");+ }++ /// A canceled partition rejects the rows routed to it, and only those: the+ /// live partitions keep their matches.+ #[cfg(not(feature = "force_hash_collisions"))]+ #[test]+ fn partitioned_canceled_partition_rejects_only_its_routed_rows() {+ let acc = make_partitioned_expr_accumulator_for_test(4);+ let keys: Vec<i32> = (0..40).collect();+ let mut partitions =+ routed_partitions(&acc, 4, &keys, |_, held| hash_map_of(held));+ partitions[0] = PartitionStatus::CanceledUnknown;+ acc.build_filter(FinalizeInput::Partitioned(partitions))+ .unwrap();++ let array: ArrayRef = Arc::new(Int32Array::from(keys.clone()));+ let mut hashes = vec![0; keys.len()];+ create_hashes(+ [&array],+ acc.repartition_random_state.random_state(),+ &mut hashes,+ )+ .unwrap();+ let expected: Vec<bool> = hashes.iter().map(|hash| *hash % 4 != 0).collect();+ assert!(expected.contains(&true) && expected.contains(&false));+ assert_matches(&acc, &keys, &expected);+ }++ #[test]+ fn partitioned_canceled_and_empty_partitions_reject_every_row() {let acc = make_partitioned_expr_accumulator_for_test(2);acc.build_filter(FinalizeInput::Partitioned(vec![@@ -1621,16 +1548,12 @@ mod tests {])).unwrap();- let expr = current_expr(&acc);- let case = case_expr(&expr);- assert_eq!(case.when_then_expr().len(), 1);- assert_literal_bool(&case.when_then_expr()[0].1, false);- assert_literal_bool(- case.else_expr().expect("expected permissive fallback"),- true,- );+ assert_literal_bool(¤t_expr(&acc), false);}+ // The routed hash lookup matches on the hash alone, so forced collisions make+ // every key match its partition's table.+ #[cfg(not(feature = "force_hash_collisions"))]#[test]fn partitioned_range_dynamic_filter_routes_with_range_expr() -> Result<()> {let mut acc = make_partitioned_expr_accumulator_for_test(4);@@ -1647,26 +1570,20 @@ mod tests {],)?);+ // No combined IN list, so each probe row is routed to its partition.+ acc.in_list_max_values = 0;+acc.build_filter(FinalizeInput::Partitioned(vec![reported(PushdownStrategy::Empty, no_bounds()),- PartitionStatus::CanceledUnknown,+ reported(in_list(&[10, 19]), no_bounds()),reported(in_list(&[20, 29]), no_bounds()),reported(in_list(&[30]), no_bounds()),]))?;let expr = current_expr(&acc);- let case = case_expr(&expr);assert!(- case.expr()- .and_then(|expr| expr.downcast_ref::<RangeExpr>())- .is_some(),- "Range routing must use RangeExpr"- );- assert_eq!(case.when_then_expr().len(), 3);- assert_literal_bool(&case.when_then_expr()[0].1, false);- assert_literal_bool(- case.else_expr().expect("expected permissive fallback"),- true,+ format!("{expr:?}").contains("routed by range over 4 partitions"),+ "{expr:?}");let batch = RecordBatch::try_new(@@ -1688,150 +1605,6 @@ mod tests {Ok(())}- #[test]- fn partitioned_range_dynamic_filter_routes_compound_nullable_keys() -> Result<()> {- let probe_schema = Arc::new(Schema::new(vec![- Field::new("probe_key", DataType::Int32, true),- Field::new("probe_tie", DataType::Int32, true),- ]));- let on_right: Vec<PhysicalExprRef> = vec![- Arc::new(Column::new("probe_key", 0)),- Arc::new(Column::new("probe_tie", 1)),- ];- let mut acc = make_accumulator_for_test(- AccumulatedBuildData::Partitioned {- partitions: vec![PartitionStatus::Pending; 4],- completed_partitions: 0,- },- on_right,- );- acc.probe_schema = Arc::clone(&probe_schema);- acc.probe_range_partitioning = Some(RangePartitioning::try_new(- [- PhysicalSortExpr::new(- Arc::clone(&acc.on_right[0]),- SortOptions::new(false, true),- ),- PhysicalSortExpr::new(- Arc::clone(&acc.on_right[1]),- SortOptions::new(false, false),- ),- ]- .into(),- vec![- SplitPoint::new(vec![- ScalarValue::Int32(None),- ScalarValue::Int32(Some(10)),- ]),- SplitPoint::new(vec![ScalarValue::Int32(None), ScalarValue::Int32(None)]),- SplitPoint::new(vec![- ScalarValue::Int32(Some(10)),- ScalarValue::Int32(None),- ]),- ],- )?);-- acc.build_filter(FinalizeInput::Partitioned(vec![- reported(PushdownStrategy::Empty, no_bounds()),- PartitionStatus::CanceledUnknown,- reported(PushdownStrategy::Empty, no_bounds()),- PartitionStatus::CanceledUnknown,- ]))?;-- let expr = current_expr(&acc);- let case = case_expr(&expr);- assert!(case.expr().is_some());- assert_eq!(case.when_then_expr().len(), 2);- assert_literal_bool(- case.else_expr().expect("expected permissive fallback"),- true,- );-- let batch = RecordBatch::try_new(- probe_schema,- vec![- Arc::new(Int32Array::from(vec![- None,- None,- None,- None,- Some(9),- Some(10),- Some(10),- Some(11),- ])),- Arc::new(Int32Array::from(vec![- Some(9),- Some(10),- Some(11),- None,- None,- Some(9),- None,- None,- ])),- ],- )?;- let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;- let result = result- .as_any()- .downcast_ref::<BooleanArray>()- .expect("dynamic filter should evaluate to BooleanArray");- assert_eq!(- result,- &BooleanArray::from(- vec![false, true, true, false, false, false, true, true,]- )- );-- Ok(())- }-- #[test]- fn partitioned_range_dynamic_filter_preserves_signed_zero_routing() -> Result<()> {- let probe_schema = Arc::new(Schema::new(vec![Field::new(- "probe_key",- DataType::Float64,- false,- )]));- let on_right: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("probe_key", 0))];- let mut acc = make_accumulator_for_test(- AccumulatedBuildData::Partitioned {- partitions: vec![PartitionStatus::Pending; 2],- completed_partitions: 0,- },- on_right,- );- acc.probe_schema = Arc::clone(&probe_schema);- acc.probe_range_partitioning = Some(RangePartitioning::try_new(- [PhysicalSortExpr::new(- Arc::clone(&acc.on_right[0]),- SortOptions::default(),- )]- .into(),- vec![SplitPoint::new(vec![ScalarValue::Float64(Some(0.0))])],- )?);-- acc.build_filter(FinalizeInput::Partitioned(vec![- PartitionStatus::CanceledUnknown,- reported(PushdownStrategy::Empty, no_bounds()),- ]))?;-- let expr = current_expr(&acc);- let batch = RecordBatch::try_new(- probe_schema,- vec![Arc::new(Float64Array::from(vec![-0.0, 0.0]))],- )?;- let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;- let result = result- .as_any()- .downcast_ref::<BooleanArray>()- .expect("dynamic filter should evaluate to BooleanArray");- assert_eq!(result, &BooleanArray::from(vec![true, false]));-- Ok(())- }-// Regression guard for the build-report lifecycle fix: on `Drop`, a stream// in `BuildReportState::ReportScheduled` still calls `report_canceled_partition`// because it cannot tell whether the coordinator has already observed the@@ -1992,10 +1765,10 @@ mod tests {);}- /// A canceled partition's build content is unknown, so it may hold a NULL key:- /// the aggregated flag must be permissive even when no partition reported one.+ /// A canceled partition holds no key, so it cannot add a NULL key that+ /// would make a null-equal join keep probe-side NULL rows.#[test]- fn partitioned_canceled_partition_widens_routed_filter() {+ fn partitioned_canceled_partition_does_not_widen_filter() {let acc = null_equal_partitioned_accumulator(2);acc.build_filter(FinalizeInput::Partitioned(vec![@@ -2005,13 +1778,7 @@ mod tests {.unwrap();let expr = current_expr(&acc);- assert_top_binary_op(&expr, Operator::Or);- assert!(- binary_expr(&expr)- .left()- .downcast_ref::<IsNullExpr>()- .is_some()- );+ assert_eq!(format!("{expr}").matches("IS NULL").count(), 0, "{expr}");}#[test]diff --git a/datafusion/physical-plan/src/repartition/range.rs b/datafusion/physical-plan/src/repartition/range.rsindex 337e3752d6..f0f64ee00a 100644--- a/datafusion/physical-plan/src/repartition/range.rs+++ b/datafusion/physical-plan/src/repartition/range.rs@@ -713,6 +713,51 @@ mod tests {)}+ /// `-0.0` sorts before `0.0`, so a `0.0` split point separates them.+ #[test]+ fn test_float_router_signed_zero() -> Result<()> {+ let split_points = make_split_points_1d(vec![ScalarValue::Float64(Some(0.0))]);+ let router = RangeRouter::try_new(&[SortOptions::default()], &split_points)?;+ let input = Arc::new(Float64Array::from(vec![-0.0, 0.0])) as ArrayRef;+ assert_routing(&router, &[input], &[0, 1], None)+ }++ /// NULLs in composite keys follow each key's own `nulls_first`.+ #[test]+ fn test_router_composite_nullable_keys() -> Result<()> {+ let split_points = vec![+ SplitPoint::new(vec![ScalarValue::Int32(None), ScalarValue::Int32(Some(10))]),+ SplitPoint::new(vec![ScalarValue::Int32(None), ScalarValue::Int32(None)]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(10)), ScalarValue::Int32(None)]),+ ];+ let sort_options = [+ SortOptions::new(false, true),+ SortOptions::new(false, false),+ ];+ let router = RangeRouter::try_new(&sort_options, &split_points)?;+ let key = Arc::new(Int32Array::from(vec![+ None,+ None,+ None,+ None,+ Some(9),+ Some(10),+ Some(10),+ Some(11),+ ])) as ArrayRef;+ let tie = Arc::new(Int32Array::from(vec![+ Some(9),+ Some(10),+ Some(11),+ None,+ None,+ Some(9),+ None,+ None,+ ])) as ArrayRef;+ assert_routing(&router, &[key, tie], &[0, 1, 1, 2, 2, 2, 3, 3], None)+ }+#[test]fn test_row_converter_strings() -> Result<()> {let split_points = make_split_points_1d(vec![