Skip to content

Apache DataFusion SQL Query Engine

fix: Remove `!=0` check from `supports_collect_by_thresholds` (#20730)

Jonathan Chen129c58fa4387
diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rs
index 29bbc8e108..88115b12f7 100644
--- a/datafusion/physical-optimizer/src/join_selection.rs
+++ b/datafusion/physical-optimizer/src/join_selection.rs
@@ -87,16 +87,18 @@ fn supports_collect_by_thresholds(
threshold_byte_size: usize,
threshold_num_rows: usize,
) -> bool {
- // Currently we do not trust the 0 value from stats, due to stats collection might have bug
- // TODO check the logic in datasource::get_statistics_with_limit()
let Ok(stats) = plan.partition_statistics(None) else {
return false;
};
+ // Stats use `Precision<T>` to represent stats, where `Absent` means unknown.
+ // `Exact(0)` and `Inexact(0)` are both valid stats, and we should not treat
+ // them as unknown, `Absent` will return None (this is in regards to why
+ // `!=0` is not checked)
if let Some(byte_size) = stats.total_byte_size.get_value() {
- *byte_size != 0 && *byte_size < threshold_byte_size
+ *byte_size < threshold_byte_size
} else if let Some(num_rows) = stats.num_rows.get_value() {
- *num_rows != 0 && *num_rows < threshold_num_rows
+ *num_rows < threshold_num_rows
} else {
false
}
diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.slt
index 4fd77be045..65f694eb37 100644
--- a/datafusion/sqllogictest/test_files/cte.slt
+++ b/datafusion/sqllogictest/test_files/cte.slt
@@ -1015,7 +1015,7 @@ logical_plan
04)----SubqueryAlias: cte
05)------TableScan: person projection=[id]
physical_plan
-01)HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(id@0, id@0)]
+01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(id@0, id@0)]
02)--DataSourceExec: partitions=1, partition_sizes=[0]
03)--DataSourceExec: partitions=1, partition_sizes=[0]