Apache DataFusion SQL Query Engine
Refactor Spark `format_string` integer conversion dispatch (#22388)
kosiew7843ab385bdc
diff --git a/datafusion/spark/src/function/string/format_string.rs b/datafusion/spark/src/function/string/format_string.rsindex 51e4ebfa7b..68b8fe5233 100644--- a/datafusion/spark/src/function/string/format_string.rs+++ b/datafusion/spark/src/function/string/format_string.rs@@ -891,23 +891,85 @@ fn unsigned_to_char(value: u64) -> Result<char> {codepoint_to_char(codepoint)}-/// Convert a non-null integer scalar to a [`char`] for the `%c` conversion.-fn integer_scalar_to_char(scalar: &ScalarValue) -> Result<char> {- match scalar {- ScalarValue::Int8(Some(value)) => signed_to_char(*value as i64),- ScalarValue::Int16(Some(value)) => signed_to_char(*value as i64),- ScalarValue::Int32(Some(value)) => signed_to_char(*value as i64),- ScalarValue::Int64(Some(value)) => signed_to_char(*value),- ScalarValue::UInt8(Some(value)) => unsigned_to_char(*value as u64),- ScalarValue::UInt16(Some(value)) => unsigned_to_char(*value as u64),- ScalarValue::UInt32(Some(value)) => unsigned_to_char(*value as u64),- ScalarValue::UInt64(Some(value)) => unsigned_to_char(*value),- _ => datafusion_common::internal_err!(- "integer_scalar_to_char expects a non-null integer scalar, got {scalar:?}"- ),- }+/// Formatting operations that differ between signed and unsigned integer+/// primitives. Signed values format as decimal for `%d` / `%s` / `%c`, but use+/// their original bit width for `%x` / `%o` via `unsigned_bits`.+trait IntegerFormatValue {+ fn unsigned_bits(self) -> u64;++ fn to_char(self) -> Result<char>;++ fn format_decimal(+ self,+ spec: &ConversionSpecifier,+ writer: &mut String,+ ) -> Result<()>;++ fn decimal_string(self) -> String;}+macro_rules! signed_integer_value {+ ($source:ty, $unsigned:ty) => {+ impl IntegerFormatValue for $source {+ fn unsigned_bits(self) -> u64 {+ (self as $unsigned) as u64+ }++ fn to_char(self) -> Result<char> {+ signed_to_char(self as i64)+ }++ fn format_decimal(+ self,+ spec: &ConversionSpecifier,+ writer: &mut String,+ ) -> Result<()> {+ spec.format_signed(writer, self as i64)+ }++ fn decimal_string(self) -> String {+ self.to_string()+ }+ }+ };+}++signed_integer_value!(i8, u8);+signed_integer_value!(i16, u16);+signed_integer_value!(i32, u32);+signed_integer_value!(i64, u64);++macro_rules! unsigned_integer_value {+ ($source:ty) => {+ impl IntegerFormatValue for $source {+ fn unsigned_bits(self) -> u64 {+ self as u64+ }++ fn to_char(self) -> Result<char> {+ unsigned_to_char(self as u64)+ }++ fn format_decimal(+ self,+ spec: &ConversionSpecifier,+ writer: &mut String,+ ) -> Result<()> {+ spec.format_unsigned(writer, self as u64)+ }++ fn decimal_string(self) -> String {+ self.to_string()+ }+ }+ };+}++unsigned_integer_value!(u8);+unsigned_integer_value!(u16);+unsigned_integer_value!(u32);+unsigned_integer_value!(u64);+impl ConversionSpecifier {/// Validates that the grouping separator flag is not used with scientific/// notation conversions, matching Java/Spark behavior which throws@@ -940,189 +1002,14 @@ impl ConversionSpecifier {_ => self.format_boolean(string, value),},- ScalarValue::Int8(Some(_))- | ScalarValue::Int16(Some(_))- | ScalarValue::Int32(Some(_))- | ScalarValue::Int64(Some(_))- | ScalarValue::UInt8(Some(_))- | ScalarValue::UInt16(Some(_))- | ScalarValue::UInt32(Some(_))- | ScalarValue::UInt64(Some(_))- if matches!(- self.conversion_type,- ConversionType::CharLower | ConversionType::CharUpper- ) =>- {- self.format_char(string, integer_scalar_to_char(value)?)- }- ScalarValue::Int8(value) => match (self.conversion_type, value) {- (ConversionType::DecInt, Some(value)) => {- self.format_signed(string, *value as i64)- }- (- ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, (*value as u8) as u64),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for Int8",- self.conversion_type- )- }- },- ScalarValue::Int16(value) => match (self.conversion_type, value) {- (ConversionType::DecInt, Some(value)) => {- self.format_signed(string, *value as i64)- }- (- ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, (*value as u16) as u64),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for Int16",- self.conversion_type- )- }- },- ScalarValue::Int32(value) => match (self.conversion_type, value) {- (ConversionType::DecInt, Some(value)) => {- self.format_signed(string, *value as i64)- }- (- ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, (*value as u32) as u64),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for Int32",- self.conversion_type- )- }- },- ScalarValue::Int64(value) => match (self.conversion_type, value) {- (ConversionType::DecInt, Some(value)) => {- self.format_signed(string, *value)- }- (- ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, *value as u64),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for Int64",- self.conversion_type- )- }- },- ScalarValue::UInt8(value) => match (self.conversion_type, value) {- (- ConversionType::DecInt- | ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, *value as u64),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for UInt8",- self.conversion_type- )- }- },- ScalarValue::UInt16(value) => match (self.conversion_type, value) {- (- ConversionType::DecInt- | ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, *value as u64),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for UInt16",- self.conversion_type- )- }- },- ScalarValue::UInt32(value) => match (self.conversion_type, value) {- (- ConversionType::DecInt- | ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, *value as u64),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for UInt32",- self.conversion_type- )- }- },- ScalarValue::UInt64(value) => match (self.conversion_type, value) {- (- ConversionType::DecInt- | ConversionType::HexIntLower- | ConversionType::HexIntUpper- | ConversionType::OctInt,- Some(value),- ) => self.format_unsigned(string, *value),- (- ConversionType::StringLower | ConversionType::StringUpper,- Some(value),- ) => self.format_string(string, &value.to_string()),- (t, None) if t.supports_integer() => self.format_string(string, "null"),- _ => {- exec_err!(- "Invalid conversion type: {:?} for UInt64",- self.conversion_type- )- }- },+ ScalarValue::Int8(value) => self.format_integer(string, value, "Int8"),+ ScalarValue::Int16(value) => self.format_integer(string, value, "Int16"),+ ScalarValue::Int32(value) => self.format_integer(string, value, "Int32"),+ ScalarValue::Int64(value) => self.format_integer(string, value, "Int64"),+ ScalarValue::UInt8(value) => self.format_integer(string, value, "UInt8"),+ ScalarValue::UInt16(value) => self.format_integer(string, value, "UInt16"),+ ScalarValue::UInt32(value) => self.format_integer(string, value, "UInt32"),+ ScalarValue::UInt64(value) => self.format_integer(string, value, "UInt64"),ScalarValue::Float16(value) => match (self.conversion_type, value) {(ConversionType::DecFloatLower@@ -1484,6 +1371,48 @@ impl ConversionSpecifier {}}+ fn format_integer<T>(+ &self,+ writer: &mut String,+ value: &Option<T>,+ type_name: &str,+ ) -> Result<()>+ where+ T: Copy + IntegerFormatValue,+ {+ let Some(value) = *value else {+ return if self.conversion_type.supports_integer() {+ self.format_string(writer, "null")+ } else {+ self.invalid_integer_conversion(type_name)+ };+ };++ match self.conversion_type {+ ConversionType::DecInt => value.format_decimal(self, writer),+ ConversionType::HexIntLower+ | ConversionType::HexIntUpper+ | ConversionType::OctInt => {+ self.format_unsigned(writer, value.unsigned_bits())+ }+ ConversionType::CharLower | ConversionType::CharUpper => {+ self.format_char(writer, value.to_char()?)+ }+ ConversionType::StringLower | ConversionType::StringUpper => {+ self.format_string(writer, &value.decimal_string())+ }+ _ => self.invalid_integer_conversion(type_name),+ }+ }++ fn invalid_integer_conversion<T>(&self, type_name: &str) -> Result<T> {+ exec_err!(+ "Invalid conversion type: {:?} for {}",+ self.conversion_type,+ type_name+ )+ }+fn format_hex_float(&self, writer: &mut String, value: f64) -> Result<()> {// Handle special cases firstlet (sign, raw_exponent, mantissa) = value.to_parts();@@ -2588,6 +2517,74 @@ mod tests {);}+ #[test]+ fn test_integer_formatting_across_widths() -> Result<()> {+ let cases = [+ (+ ScalarValue::Int8(Some(-1)),+ "%d|%x|%o|%s",+ 4,+ "-1|ff|377|-1",+ ),+ (+ ScalarValue::Int16(Some(-1)),+ "%d|%x|%o|%s",+ 4,+ "-1|ffff|177777|-1",+ ),+ (+ ScalarValue::Int32(Some(-1)),+ "%d|%x|%o|%s",+ 4,+ "-1|ffffffff|37777777777|-1",+ ),+ (+ ScalarValue::Int64(Some(-1)),+ "%d|%x|%o|%s",+ 4,+ "-1|ffffffffffffffff|1777777777777777777777|-1",+ ),+ (+ ScalarValue::UInt8(Some(255)),+ "%d|%x|%o|%s|%c",+ 5,+ "255|ff|377|255|ÿ",+ ),+ (+ ScalarValue::UInt16(Some(65535)),+ "%d|%x|%o|%s",+ 4,+ "65535|ffff|177777|65535",+ ),+ (+ ScalarValue::UInt32(Some(u32::MAX)),+ "%d|%x|%o|%s",+ 4,+ "4294967295|ffffffff|37777777777|4294967295",+ ),+ (+ ScalarValue::UInt64(Some(u64::MAX)),+ "%d|%x|%o|%s",+ 4,+ "18446744073709551615|ffffffffffffffff|1777777777777777777777|18446744073709551615",+ ),+ (+ ScalarValue::Int32(None),+ "%d|%x|%o|%s|%c",+ 5,+ "null|null|null|null|null",+ ),+ ];++ for (value, fmt, arg_count, expected) in cases {+ let data_types = vec![value.data_type(); arg_count];+ let formatter = Formatter::parse(fmt, &data_types)?;+ let args = vec![value; arg_count];+ assert_eq!(formatter.format(&args)?, expected, "{fmt}");+ }+ Ok(())+ }+#[test]fn test_insert_thousands_separator() {assert_eq!(insert_thousands_separator("1234567.89"), "1,234,567.89");