Skip to content

Apache DataFusion SQL Query Engine

refactor: Unify Spark format_string integer numeric assembly (#24480)

Goutam Adwant5091b42191f0
diff --git a/datafusion/spark/src/function/string/format_string.rs b/datafusion/spark/src/function/string/format_string.rs
index 6a65164a18..64346dd823 100644
--- a/datafusion/spark/src/function/string/format_string.rs
+++ b/datafusion/spark/src/function/string/format_string.rs
@@ -1796,9 +1796,9 @@ impl ConversionSpecifier {
fn format_signed(&self, writer: &mut String, value: i64) -> Result<()> {
let negative = value < 0;
- let abs_val = value.abs();
+ let abs_val = value.unsigned_abs();
- let (sign_prefix, sign_suffix) = if negative && self.negative_in_parentheses {
+ let (prefix, suffix) = if negative && self.negative_in_parentheses {
("(".to_owned(), ")".to_owned())
} else if negative {
("-".to_owned(), "".to_owned())
@@ -1810,60 +1810,31 @@ impl ConversionSpecifier {
("".to_owned(), "".to_owned())
};
- let mut mod_spec = *self;
- mod_spec.width = match self.width {
- NumericParam::Literal(w) => NumericParam::Literal(
- w - sign_prefix.len() as i32 - sign_suffix.len() as i32,
- ),
- _ => NumericParam::FromArgument,
- };
- let mut formatted = String::new();
- mod_spec.format_unsigned(&mut formatted, abs_val as u64)?;
- // put the sign a after any leading spaces
- let mut actual_number = &formatted[0..];
- let mut leading_spaces = &formatted[0..0];
- if let Some(first_non_space) = formatted.find(|c| c != ' ') {
- actual_number = &formatted[first_non_space..];
- leading_spaces = &formatted[0..first_non_space];
- }
- write!(
- writer,
- "{}{}{}{}",
- leading_spaces.to_owned(),
- sign_prefix,
- actual_number,
- sign_suffix
- )
- .map_err(|e| exec_datafusion_err!("Write error: {}", e))?;
+ self.format_decimal_integer(writer, abs_val, prefix, &suffix);
Ok(())
}
fn format_unsigned(&self, writer: &mut String, value: u64) -> Result<()> {
let mut s = String::new();
- let mut alt_prefix = "";
- match self.conversion_type {
+ let alt_prefix = match self.conversion_type {
ConversionType::DecInt => {
- let num_str = format!("{value}");
- s = if self.grouping_separator {
- insert_thousands_separator(&num_str)
- } else {
- num_str
- };
+ self.format_decimal_integer(writer, value, String::new(), "");
+ return Ok(());
}
ConversionType::HexIntLower => {
- alt_prefix = "0x";
write!(&mut s, "{value:x}")
.map_err(|e| exec_datafusion_err!("Write error: {}", e))?;
+ "0x"
}
ConversionType::HexIntUpper => {
- alt_prefix = "0X";
write!(&mut s, "{value:X}")
.map_err(|e| exec_datafusion_err!("Write error: {}", e))?;
+ "0X"
}
ConversionType::OctInt => {
- alt_prefix = "0";
write!(&mut s, "{value:o}")
.map_err(|e| exec_datafusion_err!("Write error: {}", e))?;
+ "0"
}
_ => {
return exec_err!(
@@ -1871,7 +1842,7 @@ impl ConversionSpecifier {
self.conversion_type
);
}
- }
+ };
let mut prefix = if self.alt_form {
alt_prefix.to_owned()
} else {
@@ -1905,6 +1876,22 @@ impl ConversionSpecifier {
Ok(())
}
+ fn format_decimal_integer(
+ &self,
+ writer: &mut String,
+ value: u64,
+ prefix: String,
+ suffix: &str,
+ ) {
+ let number = value.to_string();
+ let number = if self.grouping_separator {
+ insert_thousands_separator(&number)
+ } else {
+ number
+ };
+ self.write_numeric_parts(writer, prefix, &number, suffix, true);
+ }
+
fn format_str(&self, writer: &mut String, value: &str) -> Result<()> {
// Take care of precision, putting the truncated string in `content`
let precision: usize = match self.precision {
@@ -2573,6 +2560,80 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_numeric_final_assembly() -> Result<()> {
+ let cases = [
+ ("%(08d", ScalarValue::Int64(Some(-12)), "(000012)"),
+ ("%+08d", ScalarValue::Int64(Some(12)), "+0000012"),
+ ("%+08d", ScalarValue::Int64(Some(-12)), "-0000012"),
+ ("% 08d", ScalarValue::Int64(Some(12)), " 0000012"),
+ (
+ "%+,12d",
+ ScalarValue::Int64(Some(1_234_567)),
+ " +1,234,567",
+ ),
+ (
+ "%+,12d",
+ ScalarValue::Int64(Some(-1_234_567)),
+ " -1,234,567",
+ ),
+ (
+ "%(012.2f",
+ ScalarValue::Decimal128(Some(-123450), 10, 2),
+ "(0001234.50)",
+ ),
+ (
+ "%(012.2f",
+ ScalarValue::Decimal256(Some(i256::from(-123450)), 10, 2),
+ "(0001234.50)",
+ ),
+ (
+ "%010.2f",
+ ScalarValue::Float64(Some(f64::NAN)),
+ " NaN",
+ ),
+ (
+ "%010.2f",
+ ScalarValue::Float64(Some(f64::INFINITY)),
+ " Infinity",
+ ),
+ (
+ "%010.2f",
+ ScalarValue::Float64(Some(f64::NEG_INFINITY)),
+ " -Infinity",
+ ),
+ ("%-(8d", ScalarValue::Int64(Some(-12)), "(12) "),
+ ];
+
+ for (format, value, expected) in cases {
+ let formatter = Formatter::parse(format, &[value.data_type()])?;
+ assert_eq!(formatter.format(&[value])?, expected, "{format}");
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn test_unsigned_decimal_final_assembly_policy() -> Result<()> {
+ let cases = [
+ ("%0,12d", 1_234_567, "0001,234,567"),
+ ("%-,12d", 1_234_567, "1,234,567 "),
+ ("%12d", 1_234_567, " 1234567"),
+ ("%+08d", 42, "00000042"),
+ ("% 08d", 42, "00000042"),
+ ("%(08d", 42, "00000042"),
+ ];
+
+ for (format, value, expected) in cases {
+ let formatter = Formatter::parse(format, &[DataType::UInt64])?;
+ assert_eq!(
+ formatter.format(&[ScalarValue::UInt64(Some(value))])?,
+ expected,
+ "{format}"
+ );
+ }
+ Ok(())
+ }
+
#[test]
fn test_insert_thousands_separator() {
assert_eq!(insert_thousands_separator("1234567.89"), "1,234,567.89");
diff --git a/datafusion/sqllogictest/test_files/spark/string/format_string.slt b/datafusion/sqllogictest/test_files/spark/string/format_string.slt
index 3a100ab2d2..b0bcb8c90d 100644
--- a/datafusion/sqllogictest/test_files/spark/string/format_string.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/format_string.slt
@@ -2064,6 +2064,91 @@ SELECT format_string('Hex: %#08x', 255);
----
Hex: 0x0000ff
+# ================================
+# Shared numeric final assembly
+# ================================
+
+## Signed integer parentheses and zero padding include the suffix in the width
+query T
+SELECT format_string('[%(08d]', -12::BIGINT);
+----
+[(000012)]
+
+## Signed integer parentheses stay before right padding
+query T
+SELECT format_string('[%-(8d]', -12::BIGINT);
+----
+[(12) ]
+
+## Signed integer force-sign and zero padding
+query T
+SELECT format_string('[%+08d]', 12::BIGINT);
+----
+[+0000012]
+
+## Signed integer space-sign and zero padding
+query T
+SELECT format_string('[% 08d]', 12::BIGINT);
+----
+[ 0000012]
+
+## Negative integers keep their minus sign with force-sign or space-sign flags
+query TT
+SELECT
+ format_string('[%+08d]', -12::BIGINT),
+ format_string('[% 08d]', -12::BIGINT);
+----
+[-0000012] [-0000012]
+
+## Signed integer grouping, force-sign, and width
+query T
+SELECT format_string('[%+,12d]', 1234567::BIGINT);
+----
+[ +1,234,567]
+
+## Minimum signed integer formats without overflowing its magnitude
+query TT
+SELECT
+ format_string('%d', (-9223372036854775808)::BIGINT),
+ format_string('%(,d', (-9223372036854775808)::BIGINT);
+----
+-9223372036854775808 (9,223,372,036,854,775,808)
+
+## Signed integer array input uses the same final assembly
+query T rowsort
+SELECT format_string('[%(08d]', value)
+FROM (VALUES (-12::BIGINT), (-1234::BIGINT)) AS t(value);
+----
+[(000012)]
+[(001234)]
+
+## Unsigned integer grouping and zero padding preserve the DataFusion extension
+query T
+SELECT format_string('[%0,12d]', arrow_cast(1234567::BIGINT, 'UInt64'));
+----
+[0001,234,567]
+
+## Unsigned integer grouping and left adjustment preserve the DataFusion extension
+query T
+SELECT format_string('[%-,12d]', arrow_cast(1234567::BIGINT, 'UInt64'));
+----
+[1,234,567 ]
+
+## Negative decimal zero padding keeps the closing parenthesis last
+query T
+SELECT format_string('[%(012.2f]', -1234.50::DECIMAL(10, 2));
+----
+[(0001234.50)]
+
+## Zero padding is ignored for non-finite floating-point values
+query TTT
+SELECT
+ format_string('[%010.2f]', arrow_cast('NaN', 'Float64')),
+ format_string('[%010.2f]', arrow_cast('Infinity', 'Float64')),
+ format_string('[%010.2f]', arrow_cast('-Infinity', 'Float64'));
+----
+[ NaN] [ Infinity] [ -Infinity]
+
# ================================
# Special numeric values
# ================================