Apache DataFusion SQL Query Engine
feat: fix NTILE distribution logic (#22051)
Oleks V2bf1db5b3ad5
diff --git a/datafusion/functions-window/src/ntile.rs b/datafusion/functions-window/src/ntile.rsindex 6d9215f78e..d604b78032 100644--- a/datafusion/functions-window/src/ntile.rs+++ b/datafusion/functions-window/src/ntile.rs@@ -17,9 +17,7 @@//! `ntile` window function implementation-use crate::utils::{- get_scalar_value_from_args, get_signed_integer, get_unsigned_integer,-};+use crate::utils::{get_scalar_value_from_args, get_unsigned_integer};use arrow::datatypes::FieldRef;use datafusion_common::arrow::array::{ArrayRef, UInt64Array};use datafusion_common::arrow::datatypes::{DataType, Field};@@ -120,6 +118,7 @@ impl WindowUDFImpl for Ntile {&self,partition_evaluator_args: PartitionEvaluatorArgs,) -> Result<Box<dyn PartitionEvaluator>> {+ // check the n value is provided, guard against NTILE()let scalar_n =get_scalar_value_from_args(partition_evaluator_args.input_exprs(), 0)?.ok_or_else(|| {@@ -130,16 +129,17 @@ impl WindowUDFImpl for Ntile {return exec_err!("NTILE requires a positive integer, but finds NULL");}- if scalar_n.is_unsigned() {- let n = get_unsigned_integer(&scalar_n)?;- Ok(Box::new(NtileEvaluator { n }))- } else {- let n: i64 = get_signed_integer(&scalar_n)?;- if n <= 0 {- return exec_err!("NTILE requires a positive integer");- }- Ok(Box::new(NtileEvaluator { n: n as u64 }))+ // Works for both signed and unsigned inputs: ScalarValue::cast_to uses+ // safe=false, so negative signed values fail the cast to UInt64, and+ // routing through UInt64 also accepts values greater than i64::MAX.+ let n = get_unsigned_integer(&scalar_n)+ .map_err(|_| exec_datafusion_err!("NTILE requires a positive integer"))?;++ if n == 0 {+ return exec_err!("NTILE requires a positive integer");}++ Ok(Box::new(NtileEvaluator { n }))}fn field(&self, field_args: WindowUDFFieldArgs) -> Result<FieldRef> {let nullable = false;@@ -167,12 +167,27 @@ impl PartitionEvaluator for NtileEvaluator {_values: &[ArrayRef],num_rows: usize,) -> Result<ArrayRef> {+ // SQL NTILE distributes rows "as equally as possible": with `base = num_rows / n`+ // and `remainder = num_rows % n`, the first `remainder` buckets each contain+ // `base + 1` rows and the rest contain `base` rows. The previous formula+ // `i * n / num_rows` does not preserve those bucket sizes (e.g. NTILE(4) over+ // 10 rows yielded sizes 3,2,3,2 instead of 3,3,2,2).let num_rows = num_rows as u64;- let mut vec: Vec<u64> = Vec::new();- let n = u64::min(self.n, num_rows);+ let n = self.n;+ let mut vec: Vec<u64> = Vec::with_capacity(num_rows as usize);+ let base = num_rows / n;+ let remainder = num_rows % n;+ let large_bucket_size = base + 1;+ let large_rows = remainder * large_bucket_size;for i in 0..num_rows {- let res = i * n / num_rows;- vec.push(res + 1)+ let bucket = if i < large_rows {+ i / large_bucket_size + 1+ } else {+ // base > 0 here: i >= large_rows is only reachable when remainder < n,+ // which forces base >= 1 (otherwise large_rows would equal num_rows).+ remainder + (i - large_rows) / base + 1+ };+ vec.push(bucket);}Ok(Arc::new(UInt64Array::from(vec)))}diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex caaf22f0ad..74c2e38baa 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -3814,6 +3814,71 @@ Simpsons Lisa 710 1Simpsons Marge 990 1Simpsons Bart 2010 1+# Regression: when num_rows is not a multiple of n, the SQL standard requires the+# first `num_rows mod n` buckets to be the larger ones. Earlier the formula+# `i * n / num_rows` produced sizes 3,2,3,2 for NTILE(4) over 10 rows; the+# correct sizes are 3,3,2,2.+query I+SELECT NTILE(4) OVER (ORDER BY column1)+FROM (VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10));+----+1+1+1+2+2+2+3+3+4+4++# 7 rows / 3 buckets -> sizes 3,2,2+query I+SELECT NTILE(3) OVER (ORDER BY column1)+FROM (VALUES (1),(2),(3),(4),(5),(6),(7));+----+1+1+1+2+2+3+3++# Same regression with PARTITION BY: every 10-row partition must produce 3,3,2,2.+query III+SELECT a, b,+ NTILE(4) OVER (PARTITION BY a ORDER BY b) AS ntile_4+FROM (VALUES+ (0, 0), (0, 1), (0, 2), (0, 3), (0, 4),+ (0, 5), (0, 6), (0, 7), (0, 8), (0, 9),+ (1, 0), (1, 1), (1, 2), (1, 3), (1, 4),+ (1, 5), (1, 6), (1, 7), (1, 8), (1, 9))+ t(a, b)+ORDER BY a, b;+----+0 0 1+0 1 1+0 2 1+0 3 2+0 4 2+0 5 2+0 6 3+0 7 3+0 8 4+0 9 4+1 0 1+1 1 1+1 2 1+1 3 2+1 4 2+1 5 2+1 6 3+1 7 3+1 8 4+1 9 4+# incorrect number of parameters for ntilequery error DataFusion error: Execution error: NTILE requires a positive integer, but finds NULLSELECT