Apache DataFusion SQL Query Engine
refactor: Unify Spark format_string integer numeric assembly (#24480)
Goutam Adwant5091b42191f0
diff --git a/datafusion/spark/src/function/string/format_string.rs b/datafusion/spark/src/function/string/format_string.rsindex 6a65164a18..64346dd823 100644--- a/datafusion/spark/src/function/string/format_string.rs+++ b/datafusion/spark/src/function/string/format_string.rs@@ -1796,9 +1796,9 @@ impl ConversionSpecifier {fn format_signed(&self, writer: &mut String, value: i64) -> Result<()> {let negative = value < 0;- let abs_val = value.abs();+ let abs_val = value.unsigned_abs();- let (sign_prefix, sign_suffix) = if negative && self.negative_in_parentheses {+ let (prefix, suffix) = if negative && self.negative_in_parentheses {("(".to_owned(), ")".to_owned())} else if negative {("-".to_owned(), "".to_owned())@@ -1810,60 +1810,31 @@ impl ConversionSpecifier {("".to_owned(), "".to_owned())};- let mut mod_spec = *self;- mod_spec.width = match self.width {- NumericParam::Literal(w) => NumericParam::Literal(- w - sign_prefix.len() as i32 - sign_suffix.len() as i32,- ),- _ => NumericParam::FromArgument,- };- let mut formatted = String::new();- mod_spec.format_unsigned(&mut formatted, abs_val as u64)?;- // put the sign a after any leading spaces- let mut actual_number = &formatted[0..];- let mut leading_spaces = &formatted[0..0];- if let Some(first_non_space) = formatted.find(|c| c != ' ') {- actual_number = &formatted[first_non_space..];- leading_spaces = &formatted[0..first_non_space];- }- write!(- writer,- "{}{}{}{}",- leading_spaces.to_owned(),- sign_prefix,- actual_number,- sign_suffix- )- .map_err(|e| exec_datafusion_err!("Write error: {}", e))?;+ self.format_decimal_integer(writer, abs_val, prefix, &suffix);Ok(())}fn format_unsigned(&self, writer: &mut String, value: u64) -> Result<()> {let mut s = String::new();- let mut alt_prefix = "";- match self.conversion_type {+ let alt_prefix = match self.conversion_type {ConversionType::DecInt => {- let num_str = format!("{value}");- s = if self.grouping_separator {- insert_thousands_separator(&num_str)- } else {- num_str- };+ self.format_decimal_integer(writer, value, String::new(), "");+ return Ok(());}ConversionType::HexIntLower => {- alt_prefix = "0x";write!(&mut s, "{value:x}").map_err(|e| exec_datafusion_err!("Write error: {}", e))?;+ "0x"}ConversionType::HexIntUpper => {- alt_prefix = "0X";write!(&mut s, "{value:X}").map_err(|e| exec_datafusion_err!("Write error: {}", e))?;+ "0X"}ConversionType::OctInt => {- alt_prefix = "0";write!(&mut s, "{value:o}").map_err(|e| exec_datafusion_err!("Write error: {}", e))?;+ "0"}_ => {return exec_err!(@@ -1871,7 +1842,7 @@ impl ConversionSpecifier {self.conversion_type);}- }+ };let mut prefix = if self.alt_form {alt_prefix.to_owned()} else {@@ -1905,6 +1876,22 @@ impl ConversionSpecifier {Ok(())}+ fn format_decimal_integer(+ &self,+ writer: &mut String,+ value: u64,+ prefix: String,+ suffix: &str,+ ) {+ let number = value.to_string();+ let number = if self.grouping_separator {+ insert_thousands_separator(&number)+ } else {+ number+ };+ self.write_numeric_parts(writer, prefix, &number, suffix, true);+ }+fn format_str(&self, writer: &mut String, value: &str) -> Result<()> {// Take care of precision, putting the truncated string in `content`let precision: usize = match self.precision {@@ -2573,6 +2560,80 @@ mod tests {Ok(())}+ #[test]+ fn test_numeric_final_assembly() -> Result<()> {+ let cases = [+ ("%(08d", ScalarValue::Int64(Some(-12)), "(000012)"),+ ("%+08d", ScalarValue::Int64(Some(12)), "+0000012"),+ ("%+08d", ScalarValue::Int64(Some(-12)), "-0000012"),+ ("% 08d", ScalarValue::Int64(Some(12)), " 0000012"),+ (+ "%+,12d",+ ScalarValue::Int64(Some(1_234_567)),+ " +1,234,567",+ ),+ (+ "%+,12d",+ ScalarValue::Int64(Some(-1_234_567)),+ " -1,234,567",+ ),+ (+ "%(012.2f",+ ScalarValue::Decimal128(Some(-123450), 10, 2),+ "(0001234.50)",+ ),+ (+ "%(012.2f",+ ScalarValue::Decimal256(Some(i256::from(-123450)), 10, 2),+ "(0001234.50)",+ ),+ (+ "%010.2f",+ ScalarValue::Float64(Some(f64::NAN)),+ " NaN",+ ),+ (+ "%010.2f",+ ScalarValue::Float64(Some(f64::INFINITY)),+ " Infinity",+ ),+ (+ "%010.2f",+ ScalarValue::Float64(Some(f64::NEG_INFINITY)),+ " -Infinity",+ ),+ ("%-(8d", ScalarValue::Int64(Some(-12)), "(12) "),+ ];++ for (format, value, expected) in cases {+ let formatter = Formatter::parse(format, &[value.data_type()])?;+ assert_eq!(formatter.format(&[value])?, expected, "{format}");+ }+ Ok(())+ }++ #[test]+ fn test_unsigned_decimal_final_assembly_policy() -> Result<()> {+ let cases = [+ ("%0,12d", 1_234_567, "0001,234,567"),+ ("%-,12d", 1_234_567, "1,234,567 "),+ ("%12d", 1_234_567, " 1234567"),+ ("%+08d", 42, "00000042"),+ ("% 08d", 42, "00000042"),+ ("%(08d", 42, "00000042"),+ ];++ for (format, value, expected) in cases {+ let formatter = Formatter::parse(format, &[DataType::UInt64])?;+ assert_eq!(+ formatter.format(&[ScalarValue::UInt64(Some(value))])?,+ expected,+ "{format}"+ );+ }+ Ok(())+ }+#[test]fn test_insert_thousands_separator() {assert_eq!(insert_thousands_separator("1234567.89"), "1,234,567.89");diff --git a/datafusion/sqllogictest/test_files/spark/string/format_string.slt b/datafusion/sqllogictest/test_files/spark/string/format_string.sltindex 3a100ab2d2..b0bcb8c90d 100644--- a/datafusion/sqllogictest/test_files/spark/string/format_string.slt+++ b/datafusion/sqllogictest/test_files/spark/string/format_string.slt@@ -2064,6 +2064,91 @@ SELECT format_string('Hex: %#08x', 255);----Hex: 0x0000ff+# ================================+# Shared numeric final assembly+# ================================++## Signed integer parentheses and zero padding include the suffix in the width+query T+SELECT format_string('[%(08d]', -12::BIGINT);+----+[(000012)]++## Signed integer parentheses stay before right padding+query T+SELECT format_string('[%-(8d]', -12::BIGINT);+----+[(12) ]++## Signed integer force-sign and zero padding+query T+SELECT format_string('[%+08d]', 12::BIGINT);+----+[+0000012]++## Signed integer space-sign and zero padding+query T+SELECT format_string('[% 08d]', 12::BIGINT);+----+[ 0000012]++## Negative integers keep their minus sign with force-sign or space-sign flags+query TT+SELECT+ format_string('[%+08d]', -12::BIGINT),+ format_string('[% 08d]', -12::BIGINT);+----+[-0000012] [-0000012]++## Signed integer grouping, force-sign, and width+query T+SELECT format_string('[%+,12d]', 1234567::BIGINT);+----+[ +1,234,567]++## Minimum signed integer formats without overflowing its magnitude+query TT+SELECT+ format_string('%d', (-9223372036854775808)::BIGINT),+ format_string('%(,d', (-9223372036854775808)::BIGINT);+----+-9223372036854775808 (9,223,372,036,854,775,808)++## Signed integer array input uses the same final assembly+query T rowsort+SELECT format_string('[%(08d]', value)+FROM (VALUES (-12::BIGINT), (-1234::BIGINT)) AS t(value);+----+[(000012)]+[(001234)]++## Unsigned integer grouping and zero padding preserve the DataFusion extension+query T+SELECT format_string('[%0,12d]', arrow_cast(1234567::BIGINT, 'UInt64'));+----+[0001,234,567]++## Unsigned integer grouping and left adjustment preserve the DataFusion extension+query T+SELECT format_string('[%-,12d]', arrow_cast(1234567::BIGINT, 'UInt64'));+----+[1,234,567 ]++## Negative decimal zero padding keeps the closing parenthesis last+query T+SELECT format_string('[%(012.2f]', -1234.50::DECIMAL(10, 2));+----+[(0001234.50)]++## Zero padding is ignored for non-finite floating-point values+query TTT+SELECT+ format_string('[%010.2f]', arrow_cast('NaN', 'Float64')),+ format_string('[%010.2f]', arrow_cast('Infinity', 'Float64')),+ format_string('[%010.2f]', arrow_cast('-Infinity', 'Float64'));+----+[ NaN] [ Infinity] [ -Infinity]+# ================================# Special numeric values# ================================