Skip to content

Apache DataFusion SQL Query Engine

perf: strength reduce hash partition modulo (up to 1.16x faster) (#21900)

Daniël Heres3deeadd00207
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index f5d5d6dbd0..b4af6e2c09 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -424,7 +424,7 @@ pub struct BatchPartitioner {
enum BatchPartitionerState {
Hash {
exprs: Vec<Arc<dyn PhysicalExpr>>,
- num_partitions: usize,
+ partition_reducer: StrengthReducedU64,
hash_buffer: Vec<u64>,
indices: Vec<Vec<u32>>,
},
@@ -438,6 +438,78 @@ enum BatchPartitionerState {
/// executions and runs.
pub const REPARTITION_RANDOM_STATE: SeededRandomState = SeededRandomState::with_seed(0);
+/// Computes `value % divisor` without division in the hot loop when `divisor`
+/// is fixed for many values.
+///
+/// Hash repartitioning computes a remainder for every row. Integer division is
+/// relatively expensive, so this precomputes the strength-reduced form of the
+/// divisor: powers of two use a bit mask, and other divisors use a reciprocal
+/// multiply to recover the quotient and therefore the remainder. This is the
+/// same invariant-divisor optimization compilers use for `%` by a constant.
+#[derive(Debug, Clone, Copy)]
+enum StrengthReducedU64 {
+ PowerOfTwo { mask: u64 },
+ Reciprocal { divisor: u64, reciprocal: u128 },
+}
+
+impl StrengthReducedU64 {
+ fn new(divisor: u64) -> Self {
+ debug_assert!(divisor > 0);
+
+ if divisor.is_power_of_two() {
+ Self::PowerOfTwo { mask: divisor - 1 }
+ } else {
+ Self::Reciprocal {
+ divisor,
+ // ceil(2^128 / divisor), computed without representing 2^128
+ reciprocal: u128::MAX / u128::from(divisor) + 1,
+ }
+ }
+ }
+
+ fn partition_indices(self, hash_buffer: &[u64], indices: &mut [Vec<u32>]) {
+ match self {
+ Self::PowerOfTwo { mask } => {
+ for (index, hash) in hash_buffer.iter().enumerate() {
+ indices[(*hash & mask) as usize].push(index as u32);
+ }
+ }
+ Self::Reciprocal {
+ divisor,
+ reciprocal,
+ } => {
+ for (index, hash) in hash_buffer.iter().enumerate() {
+ let quotient = Self::quotient(*hash, reciprocal);
+ let partition = *hash - quotient * divisor;
+ indices[partition as usize].push(index as u32);
+ }
+ }
+ }
+ }
+
+ #[cfg(test)]
+ fn remainder(self, value: u64) -> u64 {
+ match self {
+ Self::PowerOfTwo { mask } => value & mask,
+ Self::Reciprocal {
+ divisor,
+ reciprocal,
+ } => value - Self::quotient(value, reciprocal) * divisor,
+ }
+ }
+
+ #[inline]
+ fn quotient(value: u64, reciprocal: u128) -> u64 {
+ let reciprocal_low = reciprocal as u64;
+ let reciprocal_high = (reciprocal >> 64) as u64;
+ let low_product = u128::from(value) * u128::from(reciprocal_low);
+ let high_product = u128::from(value) * u128::from(reciprocal_high);
+ let carry = ((high_product & u128::from(u64::MAX)) + (low_product >> 64)) >> 64;
+
+ ((high_product >> 64) + carry) as u64
+ }
+}
+
impl BatchPartitioner {
/// Create a new [`BatchPartitioner`] for hash-based repartitioning.
///
@@ -446,22 +518,29 @@ impl BatchPartitioner {
/// - `num_partitions`: Total number of output partitions.
/// - `timer`: Metric used to record time spent during repartitioning.
///
- /// # Notes
- /// This constructor cannot fail and performs no validation.
+ /// The partition count is fixed for the lifetime of the partitioner, so this
+ /// precomputes a strength-reduced reducer for `hash % num_partitions`.
+ ///
+ /// # Errors
+ /// Returns an error if `num_partitions` is zero.
pub fn new_hash_partitioner(
exprs: Vec<Arc<dyn PhysicalExpr>>,
num_partitions: usize,
timer: metrics::Time,
- ) -> Self {
- Self {
+ ) -> Result<Self> {
+ if num_partitions == 0 {
+ return internal_err!("Hash repartition requires at least one partition");
+ }
+
+ Ok(Self {
state: BatchPartitionerState::Hash {
exprs,
- num_partitions,
+ partition_reducer: StrengthReducedU64::new(num_partitions as u64),
hash_buffer: vec![],
indices: vec![vec![]; num_partitions],
},
timer,
- }
+ })
}
/// Create a new [`BatchPartitioner`] for round-robin repartitioning.
@@ -501,7 +580,8 @@ impl BatchPartitioner {
/// - `num_input_partitions`: Total number of input partitions.
///
/// # Errors
- /// Returns an error if the provided partitioning scheme is not supported.
+ /// Returns an error if the provided partitioning scheme is not supported,
+ /// or if hash partitioning is requested with zero output partitions.
pub fn try_new(
partitioning: Partitioning,
timer: metrics::Time,
@@ -510,7 +590,7 @@ impl BatchPartitioner {
) -> Result<Self> {
match partitioning {
Partitioning::Hash(exprs, num_partitions) => {
- Ok(Self::new_hash_partitioner(exprs, num_partitions, timer))
+ Self::new_hash_partitioner(exprs, num_partitions, timer)
}
Partitioning::RoundRobinBatch(num_partitions) => {
Ok(Self::new_round_robin_partitioner(
@@ -575,7 +655,7 @@ impl BatchPartitioner {
}
BatchPartitionerState::Hash {
exprs,
- num_partitions: partitions,
+ partition_reducer,
hash_buffer,
indices,
} => {
@@ -596,9 +676,7 @@ impl BatchPartitioner {
indices.iter_mut().for_each(|v| v.clear());
- for (index, hash) in hash_buffer.iter().enumerate() {
- indices[(*hash % *partitions as u64) as usize].push(index as u32);
- }
+ partition_reducer.partition_indices(hash_buffer, indices);
// Finished building index-arrays for output partitions
timer.done();
@@ -653,9 +731,9 @@ impl BatchPartitioner {
// return the number of output partitions
fn num_partitions(&self) -> usize {
- match self.state {
- BatchPartitionerState::RoundRobin { num_partitions, .. } => num_partitions,
- BatchPartitionerState::Hash { num_partitions, .. } => num_partitions,
+ match &self.state {
+ BatchPartitionerState::RoundRobin { num_partitions, .. } => *num_partitions,
+ BatchPartitionerState::Hash { indices, .. } => indices.len(),
}
}
}
@@ -1359,7 +1437,7 @@ impl RepartitionExec {
exprs.clone(),
*num_partitions,
metrics.repartition_time.clone(),
- )
+ )?
}
Partitioning::RoundRobinBatch(num_partitions) => {
BatchPartitioner::new_round_robin_partitioner(
@@ -1783,6 +1861,98 @@ mod tests {
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use insta::assert_snapshot;
+ #[test]
+ fn strength_reduced_u64_remainder_matches_modulo() {
+ let divisors = [
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 7,
+ 8,
+ 10,
+ 16,
+ 31,
+ 32,
+ 63,
+ 64,
+ 65,
+ 97,
+ u64::from(u32::MAX),
+ u64::from(u32::MAX) + 1,
+ 1_u64 << 32,
+ (1_u64 << 63) - 1,
+ 1_u64 << 63,
+ u64::MAX - 1,
+ u64::MAX,
+ ];
+ let values = [
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 31,
+ 32,
+ 33,
+ 63,
+ 64,
+ 65,
+ u64::from(u32::MAX) - 1,
+ u64::from(u32::MAX),
+ u64::from(u32::MAX) + 1,
+ (1_u64 << 32) - 1,
+ 1_u64 << 32,
+ (1_u64 << 32) + 1,
+ (1_u64 << 63) - 1,
+ 1_u64 << 63,
+ (1_u64 << 63) + 1,
+ u64::MAX - 1,
+ u64::MAX,
+ ];
+
+ for divisor in divisors {
+ let reducer = StrengthReducedU64::new(divisor);
+ for value in values {
+ assert_eq!(
+ reducer.remainder(value),
+ value % divisor,
+ "value={value} divisor={divisor}"
+ );
+ }
+
+ let mut value = 0x1234_5678_9abc_def0 ^ divisor;
+ for _ in 0..10_000 {
+ value = value
+ .wrapping_mul(6_364_136_223_846_793_005)
+ .wrapping_add(1_442_695_040_888_963_407);
+ assert_eq!(
+ reducer.remainder(value),
+ value % divisor,
+ "value={value} divisor={divisor}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn hash_partitioner_requires_nonzero_partitions() {
+ let metrics = ExecutionPlanMetricsSet::new();
+ let timer = MetricBuilder::new(&metrics).subset_time("test", 0);
+
+ let err = BatchPartitioner::new_hash_partitioner(vec![], 0, timer)
+ .err()
+ .expect("zero hash partitions should fail")
+ .to_string();
+
+ assert!(
+ err.contains("Hash repartition requires at least one partition"),
+ "actual: {err}"
+ );
+ }
+
#[tokio::test]
async fn one_to_many_round_robin() -> Result<()> {
// define input partitions