Skip to content

Apache DataFusion SQL Query Engine

fix: grouping separator for float and decimal (#20268)

Druva13642865b526
diff --git a/datafusion/spark/src/function/string/format_string.rs b/datafusion/spark/src/function/string/format_string.rs
index 8a03c7ee38..ad97841c4a 100644
--- a/datafusion/spark/src/function/string/format_string.rs
+++ b/datafusion/spark/src/function/string/format_string.rs
@@ -862,6 +862,28 @@ fn take_numeric_param(s: &str, zero: bool) -> (NumericParam, &str) {
}
impl ConversionSpecifier {
+ /// Validates that the grouping separator flag is not used with scientific
+ /// notation conversions, matching Java/Spark behavior which throws
+ /// `FormatFlagsConversionMismatchException` for `%,e` / `%,E`.
+ fn validate_grouping_separator(&self) -> Result<()> {
+ if self.grouping_separator
+ && matches!(
+ self.conversion_type,
+ ConversionType::SciFloatLower | ConversionType::SciFloatUpper
+ )
+ {
+ return exec_err!(
+ "Grouping separator ',' flag is not compatible with scientific notation conversion '{}'",
+ if self.conversion_type == ConversionType::SciFloatUpper {
+ 'E'
+ } else {
+ 'e'
+ }
+ );
+ }
+ Ok(())
+ }
+
pub fn format(&self, string: &mut String, value: &ScalarValue) -> Result<()> {
match value {
ScalarValue::Boolean(value) => match self.conversion_type {
@@ -1690,6 +1712,8 @@ impl ConversionSpecifier {
}
fn format_float(&self, writer: &mut String, value: f64) -> Result<()> {
+ self.validate_grouping_separator()?;
+
let mut prefix = String::new();
let mut suffix = String::new();
let mut number = String::new();
@@ -1770,6 +1794,9 @@ impl ConversionSpecifier {
if strip_trailing_0s {
number = trim_trailing_0s(&number).to_owned();
}
+ if self.grouping_separator {
+ number = insert_thousands_separator(&number);
+ }
}
if self.alt_form && !number.contains('.') {
number += ".";
@@ -1882,20 +1909,11 @@ impl ConversionSpecifier {
match self.conversion_type {
ConversionType::DecInt => {
let num_str = format!("{value}");
- if self.grouping_separator {
- // Add thousands separators
- let mut result = String::new();
- let chars: Vec<char> = num_str.chars().collect();
- for (i, c) in chars.iter().enumerate() {
- if i > 0 && (chars.len() - i).is_multiple_of(3) {
- result.push(',');
- }
- result.push(*c);
- }
- s = result;
+ s = if self.grouping_separator {
+ insert_thousands_separator(&num_str)
} else {
- s = num_str;
- }
+ num_str
+ };
}
ConversionType::HexIntLower => {
alt_prefix = "0x";
@@ -2000,6 +2018,8 @@ impl ConversionSpecifier {
}
fn format_decimal(&self, writer: &mut String, value: &str, scale: i64) -> Result<()> {
+ self.validate_grouping_separator()?;
+
let mut prefix = String::new();
let upper = self.conversion_type.is_upper();
@@ -2010,6 +2030,10 @@ impl ConversionSpecifier {
let decimal = BigDecimal::from_bigint(decimal, scale);
// Handle sign
+ // TODO: `negative_in_parentheses` (the `(` flag) is not implemented here.
+ // Java/Spark wrap negative values in parentheses when this flag is set
+ // (e.g. `%(,.2f` with -1234.5 → "(1,234.50)"), but this path always
+ // uses a minus sign. See `format_float` for the correct implementation.
let is_negative = decimal.sign() == Sign::Minus;
let abs_decimal = decimal.abs();
@@ -2033,7 +2057,15 @@ impl ConversionSpecifier {
let number = match self.conversion_type {
ConversionType::DecFloatLower => {
// Format as fixed-point decimal
- self.format_decimal_fixed(&abs_decimal, precision, strip_trailing_0s)?
+ let mut n = self.format_decimal_fixed(
+ &abs_decimal,
+ precision,
+ strip_trailing_0s,
+ )?;
+ if self.grouping_separator {
+ n = insert_thousands_separator(&n);
+ }
+ n
}
ConversionType::SciFloatLower => self.format_decimal_scientific(
&abs_decimal,
@@ -2062,11 +2094,15 @@ impl ConversionSpecifier {
strip_trailing_0s,
)?
} else {
- self.format_decimal_fixed(
+ let mut n = self.format_decimal_fixed(
&abs_decimal,
precision - 1 - log10_val.floor() as i32,
strip_trailing_0s,
- )?
+ )?;
+ if self.grouping_separator {
+ n = insert_thousands_separator(&n);
+ }
+ n
}
}
_ => {
@@ -2332,6 +2368,24 @@ impl FloatBits for f64 {
}
}
+/// Inserts thousands separators (`,`) into the integer part of a numeric string.
+/// For example, `"1234567.89"` becomes `"1,234,567.89"`.
+fn insert_thousands_separator(number: &str) -> String {
+ let (int_part, frac_part) = match number.find('.') {
+ Some(pos) => (&number[..pos], &number[pos..]),
+ None => (number, ""),
+ };
+ let mut result = String::with_capacity(number.len() + number.len() / 3);
+ for (i, c) in int_part.char_indices() {
+ if i > 0 && (int_part.len() - i) % 3 == 0 {
+ result.push(',');
+ }
+ result.push(c);
+ }
+ result.push_str(frac_part);
+ result
+}
+
fn trim_trailing_0s(number: &str) -> &str {
if number.contains('.') {
for (i, c) in number.chars().rev().enumerate() {
@@ -2355,6 +2409,8 @@ fn trim_trailing_0s_hex(number: &str) -> &str {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::function::utils::test::test_scalar_function;
+ use arrow::array::StringArray;
use arrow::datatypes::DataType::Utf8;
#[test]
@@ -2385,4 +2441,333 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_insert_thousands_separator() {
+ assert_eq!(insert_thousands_separator("1234567.89"), "1,234,567.89");
+ assert_eq!(insert_thousands_separator("123.45"), "123.45");
+ assert_eq!(insert_thousands_separator("1234"), "1,234");
+ assert_eq!(insert_thousands_separator("12"), "12");
+ assert_eq!(insert_thousands_separator("0.5"), "0.5");
+ assert_eq!(
+ insert_thousands_separator("1234567890.1234"),
+ "1,234,567,890.1234"
+ );
+ assert_eq!(insert_thousands_separator("1000"), "1,000");
+ assert_eq!(insert_thousands_separator("100"), "100");
+ }
+
+ #[test]
+ fn test_grouping_separator_float() -> Result<()> {
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Ok(Some("1,234,567.89")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_decimal() -> Result<()> {
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(123456789), 10, 2)),
+ ],
+ Ok(Some("1,234,567.89")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_scientific_float() -> Result<()> {
+ // %,e — Java/Spark reject grouping separator with scientific notation
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,e".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Err(DataFusionError::Execution(
+ "Grouping separator ',' flag is not compatible with scientific notation conversion 'e'".to_string(),
+ )),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,E — uppercase scientific also rejected
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,E".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Err(DataFusionError::Execution(
+ "Grouping separator ',' flag is not compatible with scientific notation conversion 'E'".to_string(),
+ )),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,.0e — precision 0 scientific with grouping also rejected
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,.0e".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Err(DataFusionError::Execution(
+ "Grouping separator ',' flag is not compatible with scientific notation conversion 'e'".to_string(),
+ )),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_compact_float() -> Result<()> {
+ // %,g with large number — triggers scientific, no commas
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,g".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Ok(Some("1.23457e+06")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,g with small number — triggers fixed-point, commas in integer part
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,g".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(12345.6))),
+ ],
+ Ok(Some("12,345.6")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,.0g — precision 0 compact with grouping (large number, scientific)
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,.0g".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Ok(Some("1e+06")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,G — uppercase compact
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,G".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Ok(Some("1.23457E+06")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_scientific_decimal() -> Result<()> {
+ // %,e on decimal — Java/Spark reject grouping separator with scientific notation
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,e".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(123456789), 10, 2)),
+ ],
+ Err(DataFusionError::Execution(
+ "Grouping separator ',' flag is not compatible with scientific notation conversion 'e'".to_string(),
+ )),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,.0e on decimal — also rejected
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,.0e".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(123456789), 10, 2)),
+ ],
+ Err(DataFusionError::Execution(
+ "Grouping separator ',' flag is not compatible with scientific notation conversion 'e'".to_string(),
+ )),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_compact_decimal() -> Result<()> {
+ // %,g on decimal — large number triggers scientific, no commas
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,g".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(123456789), 10, 2)),
+ ],
+ Ok(Some("1.23457e+06")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,g on decimal — small number triggers fixed-point, commas expected
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,g".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(1234560), 10, 2)),
+ ],
+ Ok(Some("12,345.6")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %,.0g on decimal — precision 0 compact with grouping (scientific)
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%,.0g".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(123456789), 10, 2)),
+ ],
+ Ok(Some("1e+06")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_width_sign_float() -> Result<()> {
+ // %0,15.2f — zero-pad + grouping + width
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%0,15.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Ok(Some("0001,234,567.89")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %+,15.2f — force-sign + grouping + width (space-padded)
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%+,15.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Ok(Some(" +1,234,567.89")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %-,15.2f — left-adjust + grouping + width
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%-,15.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1234567.89))),
+ ],
+ Ok(Some("1,234,567.89 ")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_width_sign_decimal() -> Result<()> {
+ // %0,15.2f — zero-pad + grouping + width on decimal
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%0,15.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(123456789), 10, 2)),
+ ],
+ Ok(Some("0001,234,567.89")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ // %+,15.2f — force-sign + grouping + width on decimal
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%+,15.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(123456789), 10, 2)),
+ ],
+ Ok(Some(" +1,234,567.89")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_parentheses_float() -> Result<()> {
+ // %(,15.2f with negative — parentheses + grouping + width
+ // Java: String.format("%(,15.2f", -1234.5) → " (1,234.50)"
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%(,15.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(-1234.5))),
+ ],
+ Ok(Some(" (1,234.50)")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn test_grouping_separator_parentheses_decimal() -> Result<()> {
+ // %(,15.2f on negative decimal — format_decimal ignores negative_in_parentheses,
+ // always uses '-'. Check TODO in fn format_decimal
+ // Java: String.format("%(,15.2f", -1234.5) → " (1,234.50)"
+ // Ours: " -1,234.50" (minus sign, no parens)
+ test_scalar_function!(
+ FormatStringFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%(,15.2f".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Decimal128(Some(-123450), 10, 2)),
+ ],
+ Ok(Some(" -1,234.50")),
+ &str,
+ Utf8,
+ StringArray
+ );
+ Ok(())
+ }
}