Apache DataFusion SQL Query Engine
Fix input handling for encoding functions & various refactors (#18754)
Jeffrey Vo8550010bd55c
diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rsindex 7eed40dcc8..7b72c264e5 100644--- a/datafusion/functions/src/encoding/inner.rs+++ b/datafusion/functions/src/encoding/inner.rs@@ -19,31 +19,30 @@use arrow::{array::{- Array, ArrayRef, BinaryArray, GenericByteArray, OffsetSizeTrait, StringArray,+ Array, ArrayRef, AsArray, BinaryArrayType, FixedSizeBinaryArray,+ GenericBinaryArray, GenericStringArray, OffsetSizeTrait,},- datatypes::{ByteArrayType, DataType},+ datatypes::DataType,};use arrow_buffer::{Buffer, OffsetBufferBuilder};use base64::{Engine as _,engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig},};-use datafusion_common::{DataFusionError, Result};-use datafusion_common::{ScalarValue, exec_err, internal_datafusion_err};use datafusion_common::{- cast::{- as_fixed_size_binary_array, as_generic_binary_array, as_generic_string_array,- },+ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err,not_impl_err, plan_err,+ types::{NativeType, logical_string},utils::take_function_args,};-use datafusion_expr::{ColumnarValue, Documentation};-use std::sync::Arc;-use std::{fmt, str::FromStr};--use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};+use datafusion_expr::{+ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ TypeSignatureClass, Volatility,+};use datafusion_macros::user_doc;use std::any::Any;+use std::fmt;+use std::sync::Arc;// Allow padding characters, but don't require them, and don't generate them.const BASE64_ENGINE: GeneralPurpose = GeneralPurpose::new(@@ -81,7 +80,17 @@ impl Default for EncodeFunc {impl EncodeFunc {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::coercible(+ vec![+ Coercion::new_implicit(+ TypeSignatureClass::Binary,+ vec![TypeSignatureClass::Native(logical_string())],+ NativeType::Binary,+ ),+ Coercion::new_exact(TypeSignatureClass::Native(logical_string())),+ ],+ Volatility::Immutable,+ ),}}}@@ -90,6 +99,7 @@ impl ScalarUDFImpl for EncodeFunc {fn as_any(&self) -> &dyn Any {self}+fn name(&self) -> &str {"encode"}@@ -99,52 +109,21 @@ impl ScalarUDFImpl for EncodeFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- use DataType::*;-- Ok(match arg_types[0] {- Utf8 => Utf8,- LargeUtf8 => LargeUtf8,- Utf8View => Utf8,- Binary => Utf8,- LargeBinary => LargeUtf8,- FixedSizeBinary(_) => Utf8,- Null => Null,- _ => {- return plan_err!(- "The encode function can only accept Utf8 or Binary or Null."- );- }- })- }-- fn invoke_with_args(- &self,- args: datafusion_expr::ScalarFunctionArgs,- ) -> Result<ColumnarValue> {- encode(&args.args)- }-- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- let [expression, format] = take_function_args(self.name(), arg_types)?;-- if format != &DataType::Utf8 {- return Err(DataFusionError::Plan("2nd argument should be Utf8".into()));+ match &arg_types[0] {+ DataType::LargeBinary => Ok(DataType::LargeUtf8),+ _ => Ok(DataType::Utf8),}+ }+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ let [expression, encoding] = take_function_args("encode", &args.args)?;+ let encoding = Encoding::try_from(encoding)?;match expression {- DataType::Utf8 | DataType::Utf8View | DataType::Null => {- Ok(vec![DataType::Utf8; 2])+ _ if expression.data_type().is_null() => {+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)))}- DataType::LargeUtf8 => Ok(vec![DataType::LargeUtf8, DataType::Utf8]),- DataType::Binary => Ok(vec![DataType::Binary, DataType::Utf8]),- DataType::LargeBinary => Ok(vec![DataType::LargeBinary, DataType::Utf8]),- DataType::FixedSizeBinary(sz) => {- Ok(vec![DataType::FixedSizeBinary(*sz), DataType::Utf8])- }- _ => plan_err!(- "1st argument should be Utf8 or Binary or Null, got {:?}",- arg_types[0]- ),+ ColumnarValue::Array(array) => encode_array(array, encoding),+ ColumnarValue::Scalar(scalar) => encode_scalar(scalar, encoding),}}@@ -178,7 +157,17 @@ impl Default for DecodeFunc {impl DecodeFunc {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::coercible(+ vec![+ Coercion::new_implicit(+ TypeSignatureClass::Binary,+ vec![TypeSignatureClass::Native(logical_string())],+ NativeType::Binary,+ ),+ Coercion::new_exact(TypeSignatureClass::Native(logical_string())),+ ],+ Volatility::Immutable,+ ),}}}@@ -187,6 +176,7 @@ impl ScalarUDFImpl for DecodeFunc {fn as_any(&self) -> &dyn Any {self}+fn name(&self) -> &str {"decode"}@@ -196,40 +186,21 @@ impl ScalarUDFImpl for DecodeFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- Ok(arg_types[0].to_owned())- }-- fn invoke_with_args(- &self,- args: datafusion_expr::ScalarFunctionArgs,- ) -> Result<ColumnarValue> {- decode(&args.args)- }-- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- if arg_types.len() != 2 {- return plan_err!(- "{} expects to get 2 arguments, but got {}",- self.name(),- arg_types.len()- );- }-- if arg_types[1] != DataType::Utf8 {- return plan_err!("2nd argument should be Utf8");+ match &arg_types[0] {+ DataType::LargeBinary => Ok(DataType::LargeBinary),+ _ => Ok(DataType::Binary),}+ }- match arg_types[0] {- DataType::Utf8 | DataType::Utf8View | DataType::Binary | DataType::Null => {- Ok(vec![DataType::Binary, DataType::Utf8])- }- DataType::LargeUtf8 | DataType::LargeBinary => {- Ok(vec![DataType::LargeBinary, DataType::Utf8])+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ let [expression, encoding] = take_function_args("decode", &args.args)?;+ let encoding = Encoding::try_from(encoding)?;+ match expression {+ _ if expression.data_type().is_null() => {+ Ok(ColumnarValue::Scalar(ScalarValue::Binary(None)))}- _ => plan_err!(- "1st argument should be Utf8 or Binary or Null, got {:?}",- arg_types[0]- ),+ ColumnarValue::Array(array) => decode_array(array, encoding),+ ColumnarValue::Scalar(scalar) => decode_scalar(scalar, encoding),}}@@ -238,339 +209,385 @@ impl ScalarUDFImpl for DecodeFunc {}}-#[derive(Debug, Copy, Clone)]-enum Encoding {- Base64,- Hex,-}--fn encode_process(value: &ColumnarValue, encoding: Encoding) -> Result<ColumnarValue> {+fn encode_scalar(value: &ScalarValue, encoding: Encoding) -> Result<ColumnarValue> {match value {- ColumnarValue::Array(a) => match a.data_type() {- DataType::Utf8 => encoding.encode_utf8_array::<i32>(a.as_ref()),- DataType::LargeUtf8 => encoding.encode_utf8_array::<i64>(a.as_ref()),- DataType::Utf8View => encoding.encode_utf8_array::<i32>(a.as_ref()),- DataType::Binary => encoding.encode_binary_array::<i32>(a.as_ref()),- DataType::LargeBinary => encoding.encode_binary_array::<i64>(a.as_ref()),- DataType::FixedSizeBinary(_) => {- encoding.encode_fixed_size_binary_array(a.as_ref())- }- other => exec_err!(- "Unsupported data type {other:?} for function encode({encoding})"- ),- },- ColumnarValue::Scalar(scalar) => {- match scalar {- ScalarValue::Utf8(a) => {- Ok(encoding.encode_scalar(a.as_ref().map(|s: &String| s.as_bytes())))- }- ScalarValue::LargeUtf8(a) => Ok(encoding- .encode_large_scalar(a.as_ref().map(|s: &String| s.as_bytes()))),- ScalarValue::Utf8View(a) => {- Ok(encoding.encode_scalar(a.as_ref().map(|s: &String| s.as_bytes())))- }- ScalarValue::Binary(a) => Ok(- encoding.encode_scalar(a.as_ref().map(|v: &Vec<u8>| v.as_slice()))- ),- ScalarValue::LargeBinary(a) => Ok(encoding- .encode_large_scalar(a.as_ref().map(|v: &Vec<u8>| v.as_slice()))),- ScalarValue::FixedSizeBinary(_, a) => Ok(- encoding.encode_scalar(a.as_ref().map(|v: &Vec<u8>| v.as_slice()))- ),- other => exec_err!(- "Unsupported data type {other:?} for function encode({encoding})"- ),- }+ ScalarValue::Binary(maybe_bytes)+ | ScalarValue::BinaryView(maybe_bytes)+ | ScalarValue::FixedSizeBinary(_, maybe_bytes) => {+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(+ maybe_bytes+ .as_ref()+ .map(|bytes| encoding.encode_bytes(bytes)),+ )))}+ ScalarValue::LargeBinary(maybe_bytes) => {+ Ok(ColumnarValue::Scalar(ScalarValue::LargeUtf8(+ maybe_bytes+ .as_ref()+ .map(|bytes| encoding.encode_bytes(bytes)),+ )))+ }+ v => internal_err!("Unexpected value for encode: {v}"),}}-fn decode_process(value: &ColumnarValue, encoding: Encoding) -> Result<ColumnarValue> {- match value {- ColumnarValue::Array(a) => match a.data_type() {- DataType::Utf8 => encoding.decode_utf8_array::<i32>(a.as_ref()),- DataType::LargeUtf8 => encoding.decode_utf8_array::<i64>(a.as_ref()),- DataType::Utf8View => encoding.decode_utf8_array::<i32>(a.as_ref()),- DataType::Binary => encoding.decode_binary_array::<i32>(a.as_ref()),- DataType::LargeBinary => encoding.decode_binary_array::<i64>(a.as_ref()),- other => exec_err!(- "Unsupported data type {other:?} for function decode({encoding})"- ),- },- ColumnarValue::Scalar(scalar) => {- match scalar {- ScalarValue::Utf8(a) => {- encoding.decode_scalar(a.as_ref().map(|s: &String| s.as_bytes()))- }- ScalarValue::LargeUtf8(a) => encoding- .decode_large_scalar(a.as_ref().map(|s: &String| s.as_bytes())),- ScalarValue::Utf8View(a) => {- encoding.decode_scalar(a.as_ref().map(|s: &String| s.as_bytes()))- }- ScalarValue::Binary(a) => {- encoding.decode_scalar(a.as_ref().map(|v: &Vec<u8>| v.as_slice()))- }- ScalarValue::LargeBinary(a) => encoding- .decode_large_scalar(a.as_ref().map(|v: &Vec<u8>| v.as_slice())),- other => exec_err!(- "Unsupported data type {other:?} for function decode({encoding})"- ),- }+fn encode_array(array: &ArrayRef, encoding: Encoding) -> Result<ColumnarValue> {+ let array = match array.data_type() {+ DataType::Binary => encoding.encode_array::<_, i32>(&array.as_binary::<i32>()),+ DataType::BinaryView => encoding.encode_array::<_, i32>(&array.as_binary_view()),+ DataType::LargeBinary => {+ encoding.encode_array::<_, i64>(&array.as_binary::<i64>())}- }+ DataType::FixedSizeBinary(_) => {+ encoding.encode_fsb_array(array.as_fixed_size_binary())+ }+ dt => {+ internal_err!("Unexpected data type for encode: {dt}")+ }+ };+ array.map(ColumnarValue::Array)}-fn hex_encode(input: &[u8]) -> String {- hex::encode(input)+fn decode_scalar(value: &ScalarValue, encoding: Encoding) -> Result<ColumnarValue> {+ match value {+ ScalarValue::Binary(maybe_bytes)+ | ScalarValue::BinaryView(maybe_bytes)+ | ScalarValue::FixedSizeBinary(_, maybe_bytes) => {+ Ok(ColumnarValue::Scalar(ScalarValue::Binary(+ maybe_bytes+ .as_ref()+ .map(|x| encoding.decode_bytes(x))+ .transpose()?,+ )))+ }+ ScalarValue::LargeBinary(maybe_bytes) => {+ Ok(ColumnarValue::Scalar(ScalarValue::LargeBinary(+ maybe_bytes+ .as_ref()+ .map(|x| encoding.decode_bytes(x))+ .transpose()?,+ )))+ }+ v => internal_err!("Unexpected value for decode: {v}"),+ }}-fn base64_encode(input: &[u8]) -> String {- BASE64_ENGINE.encode(input)+/// Estimate how many bytes are actually represented by the array; in case the+/// the array slices it's internal buffer, this returns the byte size of that slice+/// but not the byte size of the entire buffer.+///+/// This is an estimation only as it can estimate higher if null slots are non-zero+/// sized.+fn estimate_byte_data_size<O: OffsetSizeTrait>(array: &GenericBinaryArray<O>) -> usize {+ let offsets = array.value_offsets();+ // Unwraps are safe as should always have 1 element in offset buffer+ let start = *offsets.first().unwrap();+ let end = *offsets.last().unwrap();+ let data_size = end - start;+ data_size.as_usize()}-fn hex_decode(input: &[u8], buf: &mut [u8]) -> Result<usize> {- // only write input / 2 bytes to buf- let out_len = input.len() / 2;- let buf = &mut buf[..out_len];- hex::decode_to_slice(input, buf)- .map_err(|e| internal_datafusion_err!("Failed to decode from hex: {e}"))?;- Ok(out_len)+fn decode_array(array: &ArrayRef, encoding: Encoding) -> Result<ColumnarValue> {+ let array = match array.data_type() {+ DataType::Binary => {+ let array = array.as_binary::<i32>();+ encoding.decode_array::<_, i32>(&array, estimate_byte_data_size(array))+ }+ DataType::BinaryView => {+ let array = array.as_binary_view();+ // Don't know if there is a more strict upper bound we can infer+ // for view arrays byte data size.+ encoding.decode_array::<_, i32>(&array, array.get_buffer_memory_size())+ }+ DataType::LargeBinary => {+ let array = array.as_binary::<i64>();+ encoding.decode_array::<_, i64>(&array, estimate_byte_data_size(array))+ }+ DataType::FixedSizeBinary(size) => {+ let array = array.as_fixed_size_binary();+ // TODO: could we be more conservative by accounting for nulls?+ let estimate = array.len().saturating_mul(*size as usize);+ encoding.decode_fsb_array(array, estimate)+ }+ dt => {+ internal_err!("Unexpected data type for decode: {dt}")+ }+ };+ array.map(ColumnarValue::Array)}-fn base64_decode(input: &[u8], buf: &mut [u8]) -> Result<usize> {- BASE64_ENGINE- .decode_slice(input, buf)- .map_err(|e| internal_datafusion_err!("Failed to decode from base64: {e}"))+#[derive(Debug, Copy, Clone)]+enum Encoding {+ Base64,+ Hex,}-macro_rules! encode_to_array {- ($METHOD: ident, $INPUT:expr) => {{- let utf8_array: StringArray = $INPUT- .iter()- .map(|x| x.map(|x| $METHOD(x.as_ref())))- .collect();- Arc::new(utf8_array)- }};+impl fmt::Display for Encoding {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ write!(f, "{}", format!("{self:?}").to_lowercase())+ }}-fn decode_to_array<F, T: ByteArrayType>(- method: F,- input: &GenericByteArray<T>,- conservative_upper_bound_size: usize,-) -> Result<ArrayRef>-where- F: Fn(&[u8], &mut [u8]) -> Result<usize>,-{- let mut values = vec![0; conservative_upper_bound_size];- let mut offsets = OffsetBufferBuilder::new(input.len());- let mut total_bytes_decoded = 0;- for v in input {- if let Some(v) = v {- let cursor = &mut values[total_bytes_decoded..];- let decoded = method(v.as_ref(), cursor)?;- total_bytes_decoded += decoded;- offsets.push_length(decoded);- } else {- offsets.push_length(0);+impl TryFrom<&ColumnarValue> for Encoding {+ type Error = DataFusionError;++ fn try_from(encoding: &ColumnarValue) -> Result<Self> {+ let encoding = match encoding {+ ColumnarValue::Scalar(encoding) => match encoding.try_as_str().flatten() {+ Some(encoding) => encoding,+ _ => return exec_err!("Encoding must be a non-null string"),+ },+ ColumnarValue::Array(_) => {+ return not_impl_err!(+ "Encoding must be a scalar; array specified encoding is not yet supported"+ );+ }+ };+ match encoding {+ "base64" => Ok(Self::Base64),+ "hex" => Ok(Self::Hex),+ _ => {+ let options = [Self::Base64, Self::Hex]+ .iter()+ .map(|i| i.to_string())+ .collect::<Vec<_>>()+ .join(", ");+ plan_err!(+ "There is no built-in encoding named '{encoding}', currently supported encodings are: {options}"+ )+ }}}- // We reserved an upper bound size for the values buffer, but we only use the actual size- values.truncate(total_bytes_decoded);- let binary_array = BinaryArray::try_new(- offsets.finish(),- Buffer::from_vec(values),- input.nulls().cloned(),- )?;- Ok(Arc::new(binary_array))}impl Encoding {- fn encode_scalar(self, value: Option<&[u8]>) -> ColumnarValue {- ColumnarValue::Scalar(match self {- Self::Base64 => ScalarValue::Utf8(value.map(|v| BASE64_ENGINE.encode(v))),- Self::Hex => ScalarValue::Utf8(value.map(hex::encode)),- })+ fn encode_bytes(self, value: &[u8]) -> String {+ match self {+ Self::Base64 => BASE64_ENGINE.encode(value),+ Self::Hex => hex::encode(value),+ }}- fn encode_large_scalar(self, value: Option<&[u8]>) -> ColumnarValue {- ColumnarValue::Scalar(match self {- Self::Base64 => {- ScalarValue::LargeUtf8(value.map(|v| BASE64_ENGINE.encode(v)))- }- Self::Hex => ScalarValue::LargeUtf8(value.map(hex::encode)),- })+ fn decode_bytes(self, value: &[u8]) -> Result<Vec<u8>> {+ match self {+ Self::Base64 => BASE64_ENGINE.decode(value).map_err(|e| {+ exec_datafusion_err!("Failed to decode value using base64: {e}")+ }),+ Self::Hex => hex::decode(value).map_err(|e| {+ exec_datafusion_err!("Failed to decode value using hex: {e}")+ }),+ }}- fn encode_binary_array<T>(self, value: &dyn Array) -> Result<ColumnarValue>+ // OutputOffset important to ensure Large types output Large arrays+ fn encode_array<'a, InputBinaryArray, OutputOffset>(+ self,+ array: &InputBinaryArray,+ ) -> Result<ArrayRef>where- T: OffsetSizeTrait,+ InputBinaryArray: BinaryArrayType<'a>,+ OutputOffset: OffsetSizeTrait,{- let input_value = as_generic_binary_array::<T>(value)?;- let array: ArrayRef = match self {- Self::Base64 => encode_to_array!(base64_encode, input_value),- Self::Hex => encode_to_array!(hex_encode, input_value),- };- Ok(ColumnarValue::Array(array))+ match self {+ Self::Base64 => {+ let array: GenericStringArray<OutputOffset> = array+ .iter()+ .map(|x| x.map(|x| BASE64_ENGINE.encode(x)))+ .collect();+ Ok(Arc::new(array))+ }+ Self::Hex => {+ let array: GenericStringArray<OutputOffset> =+ array.iter().map(|x| x.map(hex::encode)).collect();+ Ok(Arc::new(array))+ }+ }}- fn encode_fixed_size_binary_array(self, value: &dyn Array) -> Result<ColumnarValue> {- let input_value = as_fixed_size_binary_array(value)?;- let array: ArrayRef = match self {- Self::Base64 => encode_to_array!(base64_encode, input_value),- Self::Hex => encode_to_array!(hex_encode, input_value),- };- Ok(ColumnarValue::Array(array))+ // TODO: refactor this away once https://github.com/apache/arrow-rs/pull/8993 lands+ fn encode_fsb_array(self, array: &FixedSizeBinaryArray) -> Result<ArrayRef> {+ match self {+ Self::Base64 => {+ let array: GenericStringArray<i32> = array+ .iter()+ .map(|x| x.map(|x| BASE64_ENGINE.encode(x)))+ .collect();+ Ok(Arc::new(array))+ }+ Self::Hex => {+ let array: GenericStringArray<i32> =+ array.iter().map(|x| x.map(hex::encode)).collect();+ Ok(Arc::new(array))+ }+ }}- fn encode_utf8_array<T>(self, value: &dyn Array) -> Result<ColumnarValue>+ // OutputOffset important to ensure Large types output Large arrays+ fn decode_array<'a, InputBinaryArray, OutputOffset>(+ self,+ value: &InputBinaryArray,+ approx_data_size: usize,+ ) -> Result<ArrayRef>where- T: OffsetSizeTrait,+ InputBinaryArray: BinaryArrayType<'a>,+ OutputOffset: OffsetSizeTrait,{- let input_value = as_generic_string_array::<T>(value)?;- let array: ArrayRef = match self {- Self::Base64 => encode_to_array!(base64_encode, input_value),- Self::Hex => encode_to_array!(hex_encode, input_value),- };- Ok(ColumnarValue::Array(array))- }-- fn decode_scalar(self, value: Option<&[u8]>) -> Result<ColumnarValue> {- let value = match value {- Some(value) => value,- None => return Ok(ColumnarValue::Scalar(ScalarValue::Binary(None))),- };+ fn hex_decode(input: &[u8], buf: &mut [u8]) -> Result<usize> {+ // only write input / 2 bytes to buf+ let out_len = input.len() / 2;+ let buf = &mut buf[..out_len];+ hex::decode_to_slice(input, buf)+ .map_err(|e| exec_datafusion_err!("Failed to decode from hex: {e}"))?;+ Ok(out_len)+ }- let out = match self {- Self::Base64 => BASE64_ENGINE.decode(value).map_err(|e| {- internal_datafusion_err!("Failed to decode value using base64: {e}")- })?,- Self::Hex => hex::decode(value).map_err(|e| {- internal_datafusion_err!("Failed to decode value using hex: {e}")- })?,- };+ fn base64_decode(input: &[u8], buf: &mut [u8]) -> Result<usize> {+ BASE64_ENGINE+ .decode_slice(input, buf)+ .map_err(|e| exec_datafusion_err!("Failed to decode from base64: {e}"))+ }- Ok(ColumnarValue::Scalar(ScalarValue::Binary(Some(out))))+ match self {+ Self::Base64 => {+ let upper_bound = base64::decoded_len_estimate(approx_data_size);+ delegated_decode::<_, _, OutputOffset>(base64_decode, value, upper_bound)+ }+ Self::Hex => {+ // Calculate the upper bound for decoded byte size+ // For hex encoding, each pair of hex characters (2 bytes) represents 1 byte when decoded+ // So the upper bound is half the length of the input values.+ let upper_bound = approx_data_size / 2;+ delegated_decode::<_, _, OutputOffset>(hex_decode, value, upper_bound)+ }+ }}- fn decode_large_scalar(self, value: Option<&[u8]>) -> Result<ColumnarValue> {- let value = match value {- Some(value) => value,- None => return Ok(ColumnarValue::Scalar(ScalarValue::LargeBinary(None))),- };-- let out = match self {- Self::Base64 => BASE64_ENGINE.decode(value).map_err(|e| {- internal_datafusion_err!("Failed to decode value using base64: {e}")- })?,- Self::Hex => hex::decode(value).map_err(|e| {- internal_datafusion_err!("Failed to decode value using hex: {e}")- })?,- };-- Ok(ColumnarValue::Scalar(ScalarValue::LargeBinary(Some(out))))- }+ // TODO: refactor this away once https://github.com/apache/arrow-rs/pull/8993 lands+ fn decode_fsb_array(+ self,+ value: &FixedSizeBinaryArray,+ approx_data_size: usize,+ ) -> Result<ArrayRef> {+ fn hex_decode(input: &[u8], buf: &mut [u8]) -> Result<usize> {+ // only write input / 2 bytes to buf+ let out_len = input.len() / 2;+ let buf = &mut buf[..out_len];+ hex::decode_to_slice(input, buf)+ .map_err(|e| exec_datafusion_err!("Failed to decode from hex: {e}"))?;+ Ok(out_len)+ }- fn decode_binary_array<T>(self, value: &dyn Array) -> Result<ColumnarValue>- where- T: OffsetSizeTrait,- {- let input_value = as_generic_binary_array::<T>(value)?;- let array = self.decode_byte_array(input_value)?;- Ok(ColumnarValue::Array(array))- }+ fn base64_decode(input: &[u8], buf: &mut [u8]) -> Result<usize> {+ BASE64_ENGINE+ .decode_slice(input, buf)+ .map_err(|e| exec_datafusion_err!("Failed to decode from base64: {e}"))+ }- fn decode_utf8_array<T>(self, value: &dyn Array) -> Result<ColumnarValue>- where- T: OffsetSizeTrait,- {- let input_value = as_generic_string_array::<T>(value)?;- let array = self.decode_byte_array(input_value)?;- Ok(ColumnarValue::Array(array))- }+ fn delegated_decode<DecodeFunction>(+ decode: DecodeFunction,+ input: &FixedSizeBinaryArray,+ conservative_upper_bound_size: usize,+ ) -> Result<ArrayRef>+ where+ DecodeFunction: Fn(&[u8], &mut [u8]) -> Result<usize>,+ {+ let mut values = vec![0; conservative_upper_bound_size];+ let mut offsets = OffsetBufferBuilder::new(input.len());+ let mut total_bytes_decoded = 0;+ for v in input.iter() {+ if let Some(v) = v {+ let cursor = &mut values[total_bytes_decoded..];+ let decoded = decode(v, cursor)?;+ total_bytes_decoded += decoded;+ offsets.push_length(decoded);+ } else {+ offsets.push_length(0);+ }+ }+ // We reserved an upper bound size for the values buffer, but we only use the actual size+ values.truncate(total_bytes_decoded);+ let binary_array = GenericBinaryArray::<i32>::try_new(+ offsets.finish(),+ Buffer::from_vec(values),+ input.nulls().cloned(),+ )?;+ Ok(Arc::new(binary_array))+ }- fn decode_byte_array<T: ByteArrayType>(- &self,- input_value: &GenericByteArray<T>,- ) -> Result<ArrayRef> {match self {Self::Base64 => {- let upper_bound =- base64::decoded_len_estimate(input_value.values().len());- decode_to_array(base64_decode, input_value, upper_bound)+ let upper_bound = base64::decoded_len_estimate(approx_data_size);+ delegated_decode(base64_decode, value, upper_bound)}Self::Hex => {// Calculate the upper bound for decoded byte size// For hex encoding, each pair of hex characters (2 bytes) represents 1 byte when decoded// So the upper bound is half the length of the input values.- let upper_bound = input_value.values().len() / 2;- decode_to_array(hex_decode, input_value, upper_bound)+ let upper_bound = approx_data_size / 2;+ delegated_decode(hex_decode, value, upper_bound)}}}}-impl fmt::Display for Encoding {- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {- write!(f, "{}", format!("{self:?}").to_lowercase())+fn delegated_decode<'a, DecodeFunction, InputBinaryArray, OutputOffset>(+ decode: DecodeFunction,+ input: &InputBinaryArray,+ conservative_upper_bound_size: usize,+) -> Result<ArrayRef>+where+ DecodeFunction: Fn(&[u8], &mut [u8]) -> Result<usize>,+ InputBinaryArray: BinaryArrayType<'a>,+ OutputOffset: OffsetSizeTrait,+{+ let mut values = vec![0; conservative_upper_bound_size];+ let mut offsets = OffsetBufferBuilder::new(input.len());+ let mut total_bytes_decoded = 0;+ for v in input.iter() {+ if let Some(v) = v {+ let cursor = &mut values[total_bytes_decoded..];+ let decoded = decode(v, cursor)?;+ total_bytes_decoded += decoded;+ offsets.push_length(decoded);+ } else {+ offsets.push_length(0);+ }}+ // We reserved an upper bound size for the values buffer, but we only use the actual size+ values.truncate(total_bytes_decoded);+ let binary_array = GenericBinaryArray::<OutputOffset>::try_new(+ offsets.finish(),+ Buffer::from_vec(values),+ input.nulls().cloned(),+ )?;+ Ok(Arc::new(binary_array))}-impl FromStr for Encoding {- type Err = DataFusionError;- fn from_str(name: &str) -> Result<Encoding> {- Ok(match name {- "base64" => Self::Base64,- "hex" => Self::Hex,- _ => {- let options = [Self::Base64, Self::Hex]- .iter()- .map(|i| i.to_string())- .collect::<Vec<_>>()- .join(", ");- return plan_err!(- "There is no built-in encoding named '{name}', currently supported encodings are: {options}"- );- }- })+#[cfg(test)]+mod tests {+ use arrow::array::BinaryArray;+ use arrow_buffer::OffsetBuffer;++ use super::*;++ #[test]+ fn test_estimate_byte_data_size() {+ // Offsets starting at 0, but don't count entire data buffer size+ let array = BinaryArray::new(+ OffsetBuffer::new(vec![0, 5, 10, 15].into()),+ vec![0; 100].into(),+ None,+ );+ let size = estimate_byte_data_size(&array);+ assert_eq!(size, 15);++ // Offsets starting at 0, but don't count entire data buffer size+ let array = BinaryArray::new(+ OffsetBuffer::new(vec![50, 51, 51, 60, 80, 81].into()),+ vec![0; 100].into(),+ Some(vec![true, false, false, true, true].into()),+ );+ let size = estimate_byte_data_size(&array);+ assert_eq!(size, 31);}}--/// Encodes the given data, accepts Binary, LargeBinary, Utf8, Utf8View or LargeUtf8 and returns a [`ColumnarValue`].-/// Second argument is the encoding to use.-/// Standard encodings are base64 and hex.-fn encode(args: &[ColumnarValue]) -> Result<ColumnarValue> {- let [expression, format] = take_function_args("encode", args)?;-- let encoding = match format {- ColumnarValue::Scalar(scalar) => match scalar.try_as_str() {- Some(Some(method)) => method.parse::<Encoding>(),- _ => not_impl_err!(- "Second argument to encode must be non null constant string: Encode using dynamically decided method is not yet supported. Got {scalar:?}"- ),- },- ColumnarValue::Array(_) => not_impl_err!(- "Second argument to encode must be a constant: Encode using dynamically decided method is not yet supported"- ),- }?;- encode_process(expression, encoding)-}--/// Decodes the given data, accepts Binary, LargeBinary, Utf8, Utf8View or LargeUtf8 and returns a [`ColumnarValue`].-/// Second argument is the encoding to use.-/// Standard encodings are base64 and hex.-fn decode(args: &[ColumnarValue]) -> Result<ColumnarValue> {- let [expression, format] = take_function_args("decode", args)?;-- let encoding = match format {- ColumnarValue::Scalar(scalar) => match scalar.try_as_str() {- Some(Some(method)) => method.parse::<Encoding>(),- _ => not_impl_err!(- "Second argument to decode must be a non null constant string: Decode using dynamically decided method is not yet supported. Got {scalar:?}"- ),- },- ColumnarValue::Array(_) => not_impl_err!(- "Second argument to decode must be a utf8 constant: Decode using dynamically decided method is not yet supported"- ),- }?;- decode_process(expression, encoding)-}diff --git a/datafusion/sqllogictest/test_files/encoding.slt b/datafusion/sqllogictest/test_files/encoding.sltindex 57fb76b6c8..f715f8f46a 100644--- a/datafusion/sqllogictest/test_files/encoding.slt+++ b/datafusion/sqllogictest/test_files/encoding.slt@@ -15,6 +15,32 @@# specific language governing permissions and limitations# under the License.+query T+SELECT encode(arrow_cast('tom', 'Utf8View'),'base64');+----+dG9t++query T+SELECT arrow_cast(decode(arrow_cast('dG9t', 'Utf8View'),'base64'), 'Utf8');+----+tom++query T+SELECT encode(arrow_cast('tom', 'BinaryView'),'base64');+----+dG9t++query T+SELECT arrow_cast(decode(arrow_cast('dG9t', 'BinaryView'),'base64'), 'Utf8');+----+tom++# test for hex digest+query T+select encode(digest('hello', 'sha256'), 'hex');+----+2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824+statement okCREATE TABLE test(num INT,@@ -29,23 +55,29 @@ CREATE TABLE test(;# errors-query error 1st argument should be Utf8 or Binary or Null, got Int64+query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Binary but received NativeType::Int64, DataType: Int64select encode(12, 'hex');-query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex-select encode(bin_field, 'non_encoding') from test;--query error 1st argument should be Utf8 or Binary or Null, got Int64+query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Binary but received NativeType::Int64, DataType: Int64select decode(12, 'hex');query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex-select decode(hex_field, 'non_encoding') from test;+select encode('', 'non_encoding');++query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex+select decode('', 'non_encoding');++query error DataFusion error: Execution error: Encoding must be a non-null string+select decode('', null) from test;-query error+query error DataFusion error: This feature is not implemented: Encoding must be a scalar; array specified encoding is not yet supported+select decode('', hex_field) from test;++query error DataFusion error: Error during planning: Failed to coerce arguments to satisfy a call to 'to_hex' functionselect to_hex(hex_field) from test;-query error-select arrow_cast(decode(X'8f50d3f60eae370ddbf85c86219c55108a350165', 'base64'), 'Utf8');+query error DataFusion error: Execution error: Failed to decode value using base64+select decode(X'8f50d3f60eae370ddbf85c86219c55108a350165', 'base64');# Arrays testsquery T@@ -56,13 +88,20 @@ SELECT encode(bin_field, 'hex') FROM test ORDER BY num;NULL8f50d3f60eae370ddbf85c86219c55108a350165-query T-SELECT arrow_cast(decode(base64_field, 'base64'), 'Utf8') FROM test ORDER BY num;+query TTTTTT+SELECT+ arrow_cast(decode(arrow_cast(base64_field, 'Utf8'), 'base64'), 'Utf8'),+ arrow_cast(decode(arrow_cast(base64_field, 'LargeUtf8'), 'base64'), 'Utf8'),+ arrow_cast(decode(arrow_cast(base64_field, 'Utf8View'), 'base64'), 'Utf8'),+ arrow_cast(decode(arrow_cast(base64_field, 'Binary'), 'base64'), 'Utf8'),+ arrow_cast(decode(arrow_cast(base64_field, 'LargeBinary'), 'base64'), 'Utf8'),+ arrow_cast(decode(arrow_cast(base64_field, 'BinaryView'), 'base64'), 'Utf8')+FROM test ORDER BY num;-----abc-qweqw-NULL-8f50d3f60eae370ddbf85c86219c55108a350165+abc abc abc abc abc abc+qweqw qweqw qweqw qweqw qweqw qweqw+NULL NULL NULL NULL NULL NULL+8f50d3f60eae370ddbf85c86219c55108a350165 8f50d3f60eae370ddbf85c86219c55108a350165 8f50d3f60eae370ddbf85c86219c55108a350165 8f50d3f60eae370ddbf85c86219c55108a350165 8f50d3f60eae370ddbf85c86219c55108a350165 8f50d3f60eae370ddbf85c86219c55108a350165query TSELECT arrow_cast(decode(hex_field, 'hex'), 'Utf8') FROM test ORDER BY num;@@ -90,51 +129,77 @@ select decode(encode(bin_field, 'base64'), 'base64') = X'8f50d3f60eae370ddbf85c8----true-# test for Utf8View support for encodestatement ok-CREATE TABLE test_source AS VALUES- ('Andrew', 'X'),- ('Xiangpeng', 'Xiangpeng'),- ('Raphael', 'R'),- (NULL, 'R');+drop table test+# test for Utf8View support for encodestatement okCREATE TABLE test_utf8view ASselectarrow_cast(column1, 'Utf8View') AS column1_utf8view,arrow_cast(column2, 'Utf8View') AS column2_utf8view-FROM test_source;+FROM VALUES+ ('Andrew', 'X'),+ ('Xiangpeng', 'Xiangpeng'),+ ('Raphael', 'R'),+ (NULL, 'R');-query TTTTTT+query TTTTSELECT- column1_utf8view,encode(column1_utf8view, 'base64') AS column1_base64,encode(column1_utf8view, 'hex') AS column1_hex,- column2_utf8view,encode(column2_utf8view, 'base64') AS column2_base64,encode(column2_utf8view, 'hex') AS column2_hexFROM test_utf8view;-----Andrew QW5kcmV3 416e64726577 X WA 58-Xiangpeng WGlhbmdwZW5n 5869616e6770656e67 Xiangpeng WGlhbmdwZW5n 5869616e6770656e67-Raphael UmFwaGFlbA 5261706861656c R Ug 52-NULL NULL NULL R Ug 52+QW5kcmV3 416e64726577 WA 58+WGlhbmdwZW5n 5869616e6770656e67 WGlhbmdwZW5n 5869616e6770656e67+UmFwaGFlbA 5261706861656c Ug 52+NULL NULL Ug 52-# test for hex digest-query T-select encode(digest('hello', 'sha256'), 'hex');+query TTTTTT+SELECT+ encode(arrow_cast(column1_utf8view, 'Utf8'), 'base64'),+ encode(arrow_cast(column1_utf8view, 'LargeUtf8'), 'base64'),+ encode(arrow_cast(column1_utf8view, 'Utf8View'), 'base64'),+ encode(arrow_cast(column1_utf8view, 'Binary'), 'base64'),+ encode(arrow_cast(column1_utf8view, 'LargeBinary'), 'base64'),+ encode(arrow_cast(column1_utf8view, 'BinaryView'), 'base64')+FROM test_utf8view;-----2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824+QW5kcmV3 QW5kcmV3 QW5kcmV3 QW5kcmV3 QW5kcmV3 QW5kcmV3+WGlhbmdwZW5n WGlhbmdwZW5n WGlhbmdwZW5n WGlhbmdwZW5n WGlhbmdwZW5n WGlhbmdwZW5n+UmFwaGFlbA UmFwaGFlbA UmFwaGFlbA UmFwaGFlbA UmFwaGFlbA UmFwaGFlbA+NULL NULL NULL NULL NULL NULL-# test for FixedSizeBinary support for encode+statement ok+drop table test_utf8view++# FixedSizeBinary supportstatement okCREATE TABLE test_fsb ASSELECT arrow_cast(X'0123456789ABCDEF', 'FixedSizeBinary(8)') as fsb_col;-query TT+query ??SELECT- encode(fsb_col, 'base64') AS fsb_base64,- encode(fsb_col, 'hex') AS fsb_hex-FROM test_fsb;+ decode(encode(arrow_cast(X'0123456789abcdef', 'FixedSizeBinary(8)'), 'base64'), 'base64'),+ decode(encode(arrow_cast(X'0123456789abcdef', 'FixedSizeBinary(8)'), 'hex'), 'hex');-----ASNFZ4mrze8 0123456789abcdef+0123456789abcdef 0123456789abcdef++query ??+SELECT+ decode(encode(column1, 'base64'), 'base64'),+ decode(encode(column1, 'hex'), 'hex')+FROM values+ (arrow_cast(X'0123456789abcdef', 'FixedSizeBinary(8)')),+ (arrow_cast(X'ffffffffffffffff', 'FixedSizeBinary(8)'));+----+0123456789abcdef 0123456789abcdef+ffffffffffffffff ffffffffffffffff++query error DataFusion error: Execution error: Failed to decode value using base64+select decode('invalid', 'base64');++query error DataFusion error: Execution error: Failed to decode value using hex+select decode('invalid', 'hex');