Skip to content

Apache DataFusion SQL Query Engine

Raise `hash_join_single_partition_threshold` to 4MB (#24558)

Daniël Heres7b7a9e84db1d
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index 9fc1db841e..b340ec8628 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -1775,7 +1775,7 @@ config_namespace! {
/// The maximum estimated size in bytes for one input side of a HashJoin
/// will be collected into a single partition
- pub hash_join_single_partition_threshold: usize, default = 1024 * 1024
+ pub hash_join_single_partition_threshold: usize, default = 4 * 1024 * 1024
/// The maximum estimated size in rows for one input side of a HashJoin
/// will be collected into a single partition
diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rs
index 63654ae048..5bf457ed5d 100644
--- a/datafusion/core/tests/physical_optimizer/join_selection.rs
+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs
@@ -75,6 +75,16 @@ fn get_thresholds() -> (usize, usize) {
)
}
+/// The byte size [`small_statistics`] derives from the configured threshold.
+fn small_byte_size() -> usize {
+ get_thresholds().1 / 128
+}
+
+/// The byte size [`big_statistics`] derives from the configured threshold.
+fn big_byte_size() -> usize {
+ get_thresholds().1 * 2
+}
+
/// Return statistics for small table
fn small_statistics() -> Statistics {
let (threshold_num_rows, threshold_byte_size) = get_thresholds();
@@ -258,14 +268,14 @@ async fn test_join_with_swap() {
.compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(8192)
+ Precision::Inexact(small_byte_size())
);
assert_eq!(
StatisticsContext::new()
.compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(2097152)
+ Precision::Inexact(big_byte_size())
);
}
@@ -382,14 +392,14 @@ async fn test_left_join_no_swap() {
.compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(8192)
+ Precision::Inexact(small_byte_size())
);
assert_eq!(
StatisticsContext::new()
.compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(2097152)
+ Precision::Inexact(big_byte_size())
);
}
@@ -431,14 +441,14 @@ async fn test_join_with_swap_semi() {
.compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(8192)
+ Precision::Inexact(small_byte_size())
);
assert_eq!(
StatisticsContext::new()
.compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(2097152)
+ Precision::Inexact(big_byte_size())
);
assert_eq!(original_schema, swapped_join.schema());
}
@@ -668,14 +678,14 @@ async fn test_join_with_swap_mark() {
.compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(8192)
+ Precision::Inexact(small_byte_size())
);
assert_eq!(
StatisticsContext::new()
.compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(2097152)
+ Precision::Inexact(big_byte_size())
);
assert_eq!(original_schema, swapped_join.schema());
}
@@ -794,14 +804,14 @@ async fn test_join_no_swap() {
.compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(8192)
+ Precision::Inexact(small_byte_size())
);
assert_eq!(
StatisticsContext::new()
.compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(2097152)
+ Precision::Inexact(big_byte_size())
);
}
@@ -867,14 +877,14 @@ async fn test_nl_join_with_swap(join_type: JoinType) {
.compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(8192)
+ Precision::Inexact(small_byte_size())
);
assert_eq!(
StatisticsContext::new()
.compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(2097152)
+ Precision::Inexact(big_byte_size())
);
}
@@ -938,14 +948,14 @@ async fn test_nl_join_with_swap_no_proj(join_type: JoinType) {
.compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(8192)
+ Precision::Inexact(small_byte_size())
);
assert_eq!(
StatisticsContext::new()
.compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
- Precision::Inexact(2097152)
+ Precision::Inexact(big_byte_size())
);
}
diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt
index 573fb04b34..261027cb92 100644
--- a/datafusion/sqllogictest/test_files/information_schema.slt
+++ b/datafusion/sqllogictest/test_files/information_schema.slt
@@ -324,7 +324,7 @@ datafusion.optimizer.expand_views_at_output false
datafusion.optimizer.filter_null_join_keys false
datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values 150
datafusion.optimizer.hash_join_inlist_pushdown_max_size 131072
-datafusion.optimizer.hash_join_single_partition_threshold 1048576
+datafusion.optimizer.hash_join_single_partition_threshold 4194304
datafusion.optimizer.hash_join_single_partition_threshold_rows 131072
datafusion.optimizer.join_reordering true
datafusion.optimizer.max_passes 3
@@ -484,7 +484,7 @@ datafusion.optimizer.expand_views_at_output false When set to true, if the retur
datafusion.optimizer.filter_null_join_keys false When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down.
datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values 150 Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: <https://trino.io/docs/current/admin/dynamic-filtering.html#dynamic-filter-collection-thresholds>
datafusion.optimizer.hash_join_inlist_pushdown_max_size 131072 Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` * `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins.
-datafusion.optimizer.hash_join_single_partition_threshold 1048576 The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition
+datafusion.optimizer.hash_join_single_partition_threshold 4194304 The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition
datafusion.optimizer.hash_join_single_partition_threshold_rows 131072 The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition
datafusion.optimizer.join_reordering true When set to true, the physical plan optimizer may swap join inputs based on statistics. When set to false, statistics-driven join input reordering is disabled and the original join order in the query is used.
datafusion.optimizer.max_passes 3 Number of times that the optimizer will attempt to optimize the plan
diff --git a/datafusion/sqllogictest/test_files/statistics_registry.slt b/datafusion/sqllogictest/test_files/statistics_registry.slt
index 89258bec29..f68eeac65e 100644
--- a/datafusion/sqllogictest/test_files/statistics_registry.slt
+++ b/datafusion/sqllogictest/test_files/statistics_registry.slt
@@ -162,7 +162,7 @@ statement ok
set datafusion.optimizer.use_statistics_registry = false;
statement ok
-set datafusion.optimizer.hash_join_single_partition_threshold = 1048576;
+set datafusion.optimizer.hash_join_single_partition_threshold = 4194304;
statement ok
set datafusion.optimizer.hash_join_single_partition_threshold_rows = 131072;
diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md
index e02ada03fc..dec013a88d 100644
--- a/docs/source/user-guide/configs.md
+++ b/docs/source/user-guide/configs.md
@@ -173,7 +173,7 @@ The following configuration settings are available:
| datafusion.optimizer.use_statistics_registry | false | When set to true, the physical plan optimizer uses the pluggable `StatisticsRegistry` for statistics propagation across operators. This enables more accurate cardinality estimates compared to each operator's built-in `partition_statistics`. |
| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory |
| datafusion.optimizer.enable_piecewise_merge_join | false | When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. |
-| datafusion.optimizer.hash_join_single_partition_threshold | 1048576 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition |
+| datafusion.optimizer.hash_join_single_partition_threshold | 4194304 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition |
| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition |
| datafusion.optimizer.hash_join_inlist_pushdown_max_size | 131072 | Maximum size in bytes for the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides larger than this will use hash table lookups instead. Set to 0 to always use hash table lookups. InList pushdown can be more efficient for small build sides because it can result in better statistics pruning as well as use any bloom filters present on the scan side. InList expressions are also more transparent and easier to serialize over the network in distributed uses of DataFusion. On the other hand InList pushdown requires making a copy of the data and thus adds some overhead to the build side and uses more memory. This setting is per-partition, so we may end up using `hash_join_inlist_pushdown_max_size` \* `target_partitions` memory. The default is 128kB per partition. This should allow point lookup joins (e.g. joining on a unique primary key) to use InList pushdown in most cases but avoids excessive memory usage or overhead for larger joins. |
| datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values | 150 | Maximum number of distinct values (rows) in the build side of a hash join to be pushed down as an InList expression for dynamic filtering. Build sides with more rows than this will use hash table lookups instead. Set to 0 to always use hash table lookups. This provides an additional limit beyond `hash_join_inlist_pushdown_max_size` to prevent very large IN lists that might not provide much benefit over hash table lookups. This uses the deduplicated row count once the build side has been evaluated. The default is 150 values per partition. This is inspired by Trino's `max-filter-keys-per-column` setting. See: <https://trino.io/docs/current/admin/dynamic-filtering.html#dynamic-filter-collection-thresholds> |