Apache DataFusion SQL Query Engine
perf: Optimize concat()/concat_ws() UDFs (#20317)
Neil Conway4f4e814ce331
diff --git a/datafusion/functions/src/string/concat.rs b/datafusion/functions/src/string/concat.rsindex c8da67c186..e674541253 100644--- a/datafusion/functions/src/string/concat.rs+++ b/datafusion/functions/src/string/concat.rs@@ -204,7 +204,9 @@ impl ScalarUDFImpl for ConcatFunc {DataType::Utf8View => {let string_array = as_string_view_array(array)?;- data_size += string_array.len();+ // This is an estimate; in particular, it will+ // undercount arrays of short strings (<= 12 bytes).+ data_size += string_array.total_buffer_bytes_used();let column = if array.is_nullable() {ColumnarValueRef::NullableStringViewArray(string_array)} else {diff --git a/datafusion/functions/src/string/concat_ws.rs b/datafusion/functions/src/string/concat_ws.rsindex ee62c36c04..9d3b32eedf 100644--- a/datafusion/functions/src/string/concat_ws.rs+++ b/datafusion/functions/src/string/concat_ws.rs@@ -247,6 +247,8 @@ impl ScalarUDFImpl for ConcatWsFunc {DataType::Utf8View => {let string_array = as_string_view_array(array)?;+ // This is an estimate; in particular, it will+ // undercount arrays of short strings (<= 12 bytes).data_size += string_array.total_buffer_bytes_used();let column = if array.is_nullable() {ColumnarValueRef::NullableStringViewArray(string_array)diff --git a/datafusion/functions/src/strings.rs b/datafusion/functions/src/strings.rsindex a7be3ef792..cfddf57b09 100644--- a/datafusion/functions/src/strings.rs+++ b/datafusion/functions/src/strings.rs@@ -152,43 +152,34 @@ impl StringViewArrayBuilder {}ColumnarValueRef::NullableArray(array) => {if !CHECK_VALID || array.is_valid(i) {- self.block.push_str(- std::str::from_utf8(array.value(i).as_bytes()).unwrap(),- );+ self.block.push_str(array.value(i));}}ColumnarValueRef::NullableLargeStringArray(array) => {if !CHECK_VALID || array.is_valid(i) {- self.block.push_str(- std::str::from_utf8(array.value(i).as_bytes()).unwrap(),- );+ self.block.push_str(array.value(i));}}ColumnarValueRef::NullableStringViewArray(array) => {if !CHECK_VALID || array.is_valid(i) {- self.block.push_str(- std::str::from_utf8(array.value(i).as_bytes()).unwrap(),- );+ self.block.push_str(array.value(i));}}ColumnarValueRef::NonNullableArray(array) => {- self.block- .push_str(std::str::from_utf8(array.value(i).as_bytes()).unwrap());+ self.block.push_str(array.value(i));}ColumnarValueRef::NonNullableLargeStringArray(array) => {- self.block- .push_str(std::str::from_utf8(array.value(i).as_bytes()).unwrap());+ self.block.push_str(array.value(i));}ColumnarValueRef::NonNullableStringViewArray(array) => {- self.block- .push_str(std::str::from_utf8(array.value(i).as_bytes()).unwrap());+ self.block.push_str(array.value(i));}}}pub fn append_offset(&mut self) {self.builder.append_value(&self.block);- self.block = String::new();+ self.block.clear();}pub fn finish(mut self) -> StringViewArray {