Apache DataFusion SQL Query Engine
Raise `hash_join_single_partition_threshold` to 4MB (#24558)
Daniël Heres7b7a9e84db1d
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 9fc1db841e..b340ec8628 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -1775,7 +1775,7 @@ config_namespace! {/// The maximum estimated size in bytes for one input side of a HashJoin/// will be collected into a single partition- pub hash_join_single_partition_threshold: usize, default = 1024 * 1024+ pub hash_join_single_partition_threshold: usize, default = 4 * 1024 * 1024/// The maximum estimated size in rows for one input side of a HashJoin/// will be collected into a single partitiondiff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex 63654ae048..5bf457ed5d 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -75,6 +75,16 @@ fn get_thresholds() -> (usize, usize) {)}+/// The byte size [`small_statistics`] derives from the configured threshold.+fn small_byte_size() -> usize {+ get_thresholds().1 / 128+}++/// The byte size [`big_statistics`] derives from the configured threshold.+fn big_byte_size() -> usize {+ get_thresholds().1 * 2+}+/// Return statistics for small tablefn small_statistics() -> Statistics {let (threshold_num_rows, threshold_byte_size) = get_thresholds();@@ -258,14 +268,14 @@ async fn test_join_with_swap() {.compute(swapped_join.left().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(8192)+ Precision::Inexact(small_byte_size()));assert_eq!(StatisticsContext::new().compute(swapped_join.right().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(2097152)+ Precision::Inexact(big_byte_size()));}@@ -382,14 +392,14 @@ async fn test_left_join_no_swap() {.compute(swapped_join.left().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(8192)+ Precision::Inexact(small_byte_size()));assert_eq!(StatisticsContext::new().compute(swapped_join.right().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(2097152)+ Precision::Inexact(big_byte_size()));}@@ -431,14 +441,14 @@ async fn test_join_with_swap_semi() {.compute(swapped_join.left().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(8192)+ Precision::Inexact(small_byte_size()));assert_eq!(StatisticsContext::new().compute(swapped_join.right().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(2097152)+ Precision::Inexact(big_byte_size()));assert_eq!(original_schema, swapped_join.schema());}@@ -668,14 +678,14 @@ async fn test_join_with_swap_mark() {.compute(swapped_join.left().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(8192)+ Precision::Inexact(small_byte_size()));assert_eq!(StatisticsContext::new().compute(swapped_join.right().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(2097152)+ Precision::Inexact(big_byte_size()));assert_eq!(original_schema, swapped_join.schema());}@@ -794,14 +804,14 @@ async fn test_join_no_swap() {.compute(swapped_join.left().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(8192)+ Precision::Inexact(small_byte_size()));assert_eq!(StatisticsContext::new().compute(swapped_join.right().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(2097152)+ Precision::Inexact(big_byte_size()));}@@ -867,14 +877,14 @@ async fn test_nl_join_with_swap(join_type: JoinType) {.compute(swapped_join.left().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(8192)+ Precision::Inexact(small_byte_size()));assert_eq!(StatisticsContext::new().compute(swapped_join.right().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(2097152)+ Precision::Inexact(big_byte_size()));}@@ -938,14 +948,14 @@ async fn test_nl_join_with_swap_no_proj(join_type: JoinType) {.compute(swapped_join.left().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(8192)+ Precision::Inexact(small_byte_size()));assert_eq!(StatisticsContext::new().compute(swapped_join.right().as_ref(), &StatisticsArgs::new()).unwrap().total_byte_size,- Precision::Inexact(2097152)+ Precision::Inexact(big_byte_size()));}diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex 573fb04b34..261027cb92 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -324,7 +324,7 @@ datafusion.optimizer.expand_views_at_output falsedatafusion.optimizer.filter_null_join_keys falsedatafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values 150datafusion.optimizer.hash_join_inlist_pushdown_max_size 131072-datafusion.optimizer.hash_join_single_partition_threshold 1048576+datafusion.optimizer.hash_join_single_partition_threshold 4194304datafusion.optimizer.hash_join_single_partition_threshold_rows 131072datafusion.optimizer.join_reordering truedatafusion.optimizer.max_passes 3@@ -484,7 +484,7 @@ datafusion.optimizer.expand_views_at_output false When set to true, if the returdatafusion.optimizer.filter_null_join_keys false When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down.datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values 150 Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: <https://trino.io/docs/current/admin/dynamic-filtering.html#dynamic-filter-collection-thresholds>datafusion.optimizer.hash_join_inlist_pushdown_max_size 131072 Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` * `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins.-datafusion.optimizer.hash_join_single_partition_threshold 1048576 The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition+datafusion.optimizer.hash_join_single_partition_threshold 4194304 The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partitiondatafusion.optimizer.hash_join_single_partition_threshold_rows 131072 The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partitiondatafusion.optimizer.join_reordering true When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used.datafusion.optimizer.max_passes 3 Number of times that the optimizer will attempt to optimize the plandiff --git a/datafusion/sqllogictest/test_files/statistics_registry.slt b/datafusion/sqllogictest/test_files/statistics_registry.sltindex 89258bec29..f68eeac65e 100644--- a/datafusion/sqllogictest/test_files/statistics_registry.slt+++ b/datafusion/sqllogictest/test_files/statistics_registry.slt@@ -162,7 +162,7 @@ statement okset datafusion.optimizer.use_statistics_registry = false;statement ok-set datafusion.optimizer.hash_join_single_partition_threshold = 1048576;+set datafusion.optimizer.hash_join_single_partition_threshold = 4194304;statement okset datafusion.optimizer.hash_join_single_partition_threshold_rows = 131072;diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex e02ada03fc..dec013a88d 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -173,7 +173,7 @@ The following configuration settings are available:| datafusion.optimizer.use_statistics_registry | false | When set to true, the physical plan optimizer uses the pluggable `StatisticsRegistry` for statistics propagation across operators. This enables more accurate cardinality estimates compared to each operator's built-in `partition_statistics`. || datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory || datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. |-| datafusion.optimizer.hash_join_single_partition_threshold | 1048576 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition |+| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition || datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition || datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. || datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: <https://trino.io/docs/current/admin/dynamic-filtering.html#dynamic-filter-collection-thresholds> |