Apache DataFusion SQL Query Engine
fix: Remove `!=0` check from `supports_collect_by_thresholds` (#20730)
Jonathan Chen129c58fa4387
diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rsindex 29bbc8e108..88115b12f7 100644--- a/datafusion/physical-optimizer/src/join_selection.rs+++ b/datafusion/physical-optimizer/src/join_selection.rs@@ -87,16 +87,18 @@ fn supports_collect_by_thresholds(threshold_byte_size: usize,threshold_num_rows: usize,) -> bool {- // Currently we do not trust the 0 value from stats, due to stats collection might have bug- // TODO check the logic in datasource::get_statistics_with_limit()let Ok(stats) = plan.partition_statistics(None) else {return false;};+ // Stats use `Precision<T>` to represent stats, where `Absent` means unknown.+ // `Exact(0)` and `Inexact(0)` are both valid stats, and we should not treat+ // them as unknown, `Absent` will return None (this is in regards to why+ // `!=0` is not checked)if let Some(byte_size) = stats.total_byte_size.get_value() {- *byte_size != 0 && *byte_size < threshold_byte_size+ *byte_size < threshold_byte_size} else if let Some(num_rows) = stats.num_rows.get_value() {- *num_rows != 0 && *num_rows < threshold_num_rows+ *num_rows < threshold_num_rows} else {false}diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex 4fd77be045..65f694eb37 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -1015,7 +1015,7 @@ logical_plan04)----SubqueryAlias: cte05)------TableScan: person projection=[id]physical_plan-01)HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(id@0, id@0)]+01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(id@0, id@0)]02)--DataSourceExec: partitions=1, partition_sizes=[0]03)--DataSourceExec: partitions=1, partition_sizes=[0]