Apache DataFusion SQL Query Engine
Allow base64 encoding of fixedsizebinary arrays (#18950)
Max Burke16fa35fa2597
diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rsindex cb3f45b60a..d60c39a25d 100644--- a/datafusion/functions/src/encoding/inner.rs+++ b/datafusion/functions/src/encoding/inner.rs@@ -31,7 +31,9 @@ use base64::{use datafusion_common::{DataFusionError, Result};use datafusion_common::{ScalarValue, exec_err, internal_datafusion_err};use datafusion_common::{- cast::{as_generic_binary_array, as_generic_string_array},+ cast::{+ as_fixed_size_binary_array, as_generic_binary_array, as_generic_string_array,+ },not_impl_err, plan_err,utils::take_function_args,};@@ -105,6 +107,7 @@ impl ScalarUDFImpl for EncodeFunc {Utf8View => Utf8,Binary => Utf8,LargeBinary => LargeUtf8,+ FixedSizeBinary(_) => Utf8,Null => Null,_ => {return plan_err!(@@ -135,6 +138,9 @@ impl ScalarUDFImpl for EncodeFunc {DataType::LargeUtf8 => Ok(vec![DataType::LargeUtf8, DataType::Utf8]),DataType::Binary => Ok(vec![DataType::Binary, DataType::Utf8]),DataType::LargeBinary => Ok(vec![DataType::LargeBinary, DataType::Utf8]),+ DataType::FixedSizeBinary(sz) => {+ Ok(vec![DataType::FixedSizeBinary(*sz), DataType::Utf8])+ }_ => plan_err!("1st argument should be Utf8 or Binary or Null, got {:?}",arg_types[0]@@ -246,6 +252,9 @@ fn encode_process(value: &ColumnarValue, encoding: Encoding) -> Result<ColumnarVDataType::Utf8View => encoding.encode_utf8_array::<i32>(a.as_ref()),DataType::Binary => encoding.encode_binary_array::<i32>(a.as_ref()),DataType::LargeBinary => encoding.encode_binary_array::<i64>(a.as_ref()),+ DataType::FixedSizeBinary(_) => {+ encoding.encode_fixed_size_binary_array(a.as_ref())+ }other => exec_err!("Unsupported data type {other:?} for function encode({encoding})"),@@ -265,6 +274,9 @@ fn encode_process(value: &ColumnarValue, encoding: Encoding) -> Result<ColumnarV),ScalarValue::LargeBinary(a) => Ok(encoding.encode_large_scalar(a.as_ref().map(|v: &Vec<u8>| v.as_slice()))),+ ScalarValue::FixedSizeBinary(_, a) => Ok(+ encoding.encode_scalar(a.as_ref().map(|v: &Vec<u8>| v.as_slice()))+ ),other => exec_err!("Unsupported data type {other:?} for function encode({encoding})"),@@ -401,6 +413,15 @@ impl Encoding {Ok(ColumnarValue::Array(array))}+ fn encode_fixed_size_binary_array(self, value: &dyn Array) -> Result<ColumnarValue> {+ let input_value = as_fixed_size_binary_array(value)?;+ let array: ArrayRef = match self {+ Self::Base64 => encode_to_array!(base64_encode, input_value),+ Self::Hex => encode_to_array!(hex_encode, input_value),+ };+ Ok(ColumnarValue::Array(array))+ }+fn encode_utf8_array<T>(self, value: &dyn Array) -> Result<ColumnarValue>whereT: OffsetSizeTrait,@@ -553,3 +574,29 @@ fn decode(args: &[ColumnarValue]) -> Result<ColumnarValue> {}?;decode_process(expression, encoding)}++#[cfg(test)]+mod tests {+ #[test]+ fn test_encode_fsb() {+ use super::*;++ let value = vec![0u8; 16];+ let array = arrow::array::FixedSizeBinaryArray::try_from_sparse_iter_with_size(+ vec![Some(value)].into_iter(),+ 16,+ )+ .unwrap();+ let value = ColumnarValue::Array(Arc::new(array));++ let ColumnarValue::Array(result) =+ encode_process(&value, Encoding::Base64).unwrap()+ else {+ panic!("unexpected value");+ };++ let string_array = result.as_any().downcast_ref::<StringArray>().unwrap();+ let result_value = string_array.value(0);+ assert_eq!(result_value, "AAAAAAAAAAAAAAAAAAAAAA");+ }+}diff --git a/datafusion/sqllogictest/test_files/encoding.slt b/datafusion/sqllogictest/test_files/encoding.sltindex 300294f6e1..57fb76b6c8 100644--- a/datafusion/sqllogictest/test_files/encoding.slt+++ b/datafusion/sqllogictest/test_files/encoding.slt@@ -125,3 +125,16 @@ query Tselect encode(digest('hello', 'sha256'), 'hex');----2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824++# test for FixedSizeBinary support for encode+statement ok+CREATE TABLE test_fsb AS+SELECT arrow_cast(X'0123456789ABCDEF', 'FixedSizeBinary(8)') as fsb_col;++query TT+SELECT+ encode(fsb_col, 'base64') AS fsb_base64,+ encode(fsb_col, 'hex') AS fsb_hex+FROM test_fsb;+----+ASNFZ4mrze8 0123456789abcdef