Apache DataFusion SQL Query Engine
feat: support GroupsAccumulator for first_value and last_value with string/binary types (#21090)
UBarney587f4c0c7435
diff --git a/benchmarks/queries/clickbench/README.md b/benchmarks/queries/clickbench/README.mdindex 877ea0e0c3..cead49586b 100644--- a/benchmarks/queries/clickbench/README.md+++ b/benchmarks/queries/clickbench/README.md@@ -228,6 +228,22 @@ Results look likeElapsed 30.195 seconds.```++### Q9-Q12: FIRST_VALUE Aggregation Performance++These queries test the performance of the `FIRST_VALUE` aggregation function with different data types and grouping cardinalities.++| Query | `FIRST_VALUE` Column | Column Type | Group By Column | Group By Type | Number of Groups |+|-------|----------------------|-------------|-----------------|---------------|------------------|+| Q9 | `URL` | `Utf8` | `UserID` | `Int64` | 17,630,976 |+| Q10 | `URL` | `Utf8` | `OS` | `Int16` | 91 |+| Q11 | `WatchID` | `Int64` | `UserID` | `Int64` | 17,630,976 |+| Q12 | `WatchID` | `Int64` | `OS` | `Int16` | 91 |+++++## Data NotesHere are some interesting statistics about the data used in the queriesdiff --git a/benchmarks/queries/clickbench/extended/q10.sql b/benchmarks/queries/clickbench/extended/q10.sqlnew file mode 100644index 0000000000..d6019de178--- /dev/null+++ b/benchmarks/queries/clickbench/extended/q10.sql@@ -0,0 +1,8 @@+-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591+-- set datafusion.execution.parquet.binary_as_string = true++SELECT MAX(len) FROM (+ SELECT LENGTH(FIRST_VALUE("URL" ORDER BY "EventTime")) as len+ FROM hits+ GROUP BY "OS"+);diff --git a/benchmarks/queries/clickbench/extended/q11.sql b/benchmarks/queries/clickbench/extended/q11.sqlnew file mode 100644index 0000000000..bca38f836b--- /dev/null+++ b/benchmarks/queries/clickbench/extended/q11.sql@@ -0,0 +1,8 @@+-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591+-- set datafusion.execution.parquet.binary_as_string = true++SELECT MAX(fv) FROM (+ SELECT FIRST_VALUE("WatchID" ORDER BY "EventTime") as fv+ FROM hits+ GROUP BY "UserID"+);diff --git a/benchmarks/queries/clickbench/extended/q12.sql b/benchmarks/queries/clickbench/extended/q12.sqlnew file mode 100644index 0000000000..fa062ac1f5--- /dev/null+++ b/benchmarks/queries/clickbench/extended/q12.sql@@ -0,0 +1,8 @@+-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591+-- set datafusion.execution.parquet.binary_as_string = true++SELECT MAX(fv) FROM (+ SELECT FIRST_VALUE("WatchID" ORDER BY "EventTime") as fv+ FROM hits+ GROUP BY "OS"+);diff --git a/benchmarks/queries/clickbench/extended/q8.sql b/benchmarks/queries/clickbench/extended/q8.sqlnew file mode 100644index 0000000000..e580807841--- /dev/null+++ b/benchmarks/queries/clickbench/extended/q8.sql@@ -0,0 +1,4 @@+-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591+-- set datafusion.execution.parquet.binary_as_string = true++SELECT "RegionID", "UserAgent", "OS", AVG(to_timestamp("ResponseEndTiming")-to_timestamp("ResponseStartTiming")) as avg_response_time, AVG(to_timestamp("ResponseEndTiming")-to_timestamp("ConnectTiming")) as avg_latency FROM hits GROUP BY "RegionID", "UserAgent", "OS" ORDER BY avg_latency DESC limit 10;\ No newline at end of filediff --git a/benchmarks/queries/clickbench/extended/q9.sql b/benchmarks/queries/clickbench/extended/q9.sqlnew file mode 100644index 0000000000..53952ebec2--- /dev/null+++ b/benchmarks/queries/clickbench/extended/q9.sql@@ -0,0 +1,8 @@+-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591+-- set datafusion.execution.parquet.binary_as_string = true++SELECT MAX(len) FROM (+ SELECT LENGTH(FIRST_VALUE("URL" ORDER BY "EventTime")) as len+ FROM hits+ GROUP BY "UserID"+);diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rsindex 91964eba1a..8e6bf9205b 100644--- a/datafusion/functions-aggregate/src/first_last.rs+++ b/datafusion/functions-aggregate/src/first_last.rs@@ -22,11 +22,8 @@ use std::hash::Hash;use std::mem::size_of_val;use std::sync::Arc;-use arrow::array::{- Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, BooleanBufferBuilder,- PrimitiveArray,-};-use arrow::buffer::{BooleanBuffer, NullBuffer};+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder};+use arrow::buffer::BooleanBuffer;use arrow::compute::{self, LexicographicalComparator, SortColumn, SortOptions};use arrow::datatypes::{DataType, Date32Type, Date64Type, Decimal32Type, Decimal64Type, Decimal128Type,@@ -52,6 +49,10 @@ use datafusion_functions_aggregate_common::utils::get_sort_options;use datafusion_macros::user_doc;use datafusion_physical_expr_common::sort_expr::LexOrdering;+mod state;++use state::{BytesValueState, PrimitiveValueState, ValueState};+create_func!(FirstValue, first_value_udaf);create_func!(LastValue, last_value_udaf);@@ -75,6 +76,142 @@ pub fn last_value(expression: Expr, order_by: Vec<SortExpr>) -> Expr {.unwrap()}+fn create_groups_accumulator_helper<S: ValueState + 'static>(+ args: &AccumulatorArgs,+ is_first: bool,+ state: S,+) -> Result<Box<dyn GroupsAccumulator>> {+ let Some(ordering) = LexOrdering::new(args.order_bys.to_vec()) else {+ return internal_err!("Groups accumulator must have an ordering.");+ };++ let ordering_dtypes = ordering+ .iter()+ .map(|e| e.expr.data_type(args.schema))+ .collect::<Result<Vec<_>>>()?;++ Ok(Box::new(FirstLastGroupsAccumulator::try_new(+ state,+ ordering,+ args.ignore_nulls,+ &ordering_dtypes,+ is_first,+ )?))+}++fn create_groups_accumulator(+ args: &AccumulatorArgs,+ is_first: bool,+ function_name: &str,+) -> Result<Box<dyn GroupsAccumulator>> {+ let data_type = args.return_field.data_type();++ macro_rules! instantiate_primitive {+ ($t:ty) => {+ create_groups_accumulator_helper(+ args,+ is_first,+ PrimitiveValueState::<$t>::new(data_type.clone()),+ )+ };+ }++ match data_type {+ DataType::Int8 => instantiate_primitive!(Int8Type),+ DataType::Int16 => instantiate_primitive!(Int16Type),+ DataType::Int32 => instantiate_primitive!(Int32Type),+ DataType::Int64 => instantiate_primitive!(Int64Type),+ DataType::UInt8 => instantiate_primitive!(UInt8Type),+ DataType::UInt16 => instantiate_primitive!(UInt16Type),+ DataType::UInt32 => instantiate_primitive!(UInt32Type),+ DataType::UInt64 => instantiate_primitive!(UInt64Type),+ DataType::Float16 => instantiate_primitive!(Float16Type),+ DataType::Float32 => instantiate_primitive!(Float32Type),+ DataType::Float64 => instantiate_primitive!(Float64Type),++ DataType::Decimal32(_, _) => instantiate_primitive!(Decimal32Type),+ DataType::Decimal64(_, _) => instantiate_primitive!(Decimal64Type),+ DataType::Decimal128(_, _) => instantiate_primitive!(Decimal128Type),+ DataType::Decimal256(_, _) => instantiate_primitive!(Decimal256Type),++ DataType::Timestamp(TimeUnit::Second, _) => {+ instantiate_primitive!(TimestampSecondType)+ }+ DataType::Timestamp(TimeUnit::Millisecond, _) => {+ instantiate_primitive!(TimestampMillisecondType)+ }+ DataType::Timestamp(TimeUnit::Microsecond, _) => {+ instantiate_primitive!(TimestampMicrosecondType)+ }+ DataType::Timestamp(TimeUnit::Nanosecond, _) => {+ instantiate_primitive!(TimestampNanosecondType)+ }++ DataType::Date32 => instantiate_primitive!(Date32Type),+ DataType::Date64 => instantiate_primitive!(Date64Type),+ DataType::Time32(TimeUnit::Second) => instantiate_primitive!(Time32SecondType),+ DataType::Time32(TimeUnit::Millisecond) => {+ instantiate_primitive!(Time32MillisecondType)+ }+ DataType::Time64(TimeUnit::Microsecond) => {+ instantiate_primitive!(Time64MicrosecondType)+ }+ DataType::Time64(TimeUnit::Nanosecond) => {+ instantiate_primitive!(Time64NanosecondType)+ }++ DataType::Utf8+ | DataType::LargeUtf8+ | DataType::Utf8View+ | DataType::Binary+ | DataType::LargeBinary+ | DataType::BinaryView => create_groups_accumulator_helper(+ args,+ is_first,+ BytesValueState::try_new(data_type.clone())?,+ ),++ _ => internal_err!(+ "GroupsAccumulator not supported for {}({})",+ function_name,+ data_type+ ),+ }+}++fn groups_accumulator_supported(args: &AccumulatorArgs) -> bool {+ use DataType::*;+ !args.order_bys.is_empty()+ && matches!(+ args.return_field.data_type(),+ Int8 | Int16+ | Int32+ | Int64+ | UInt8+ | UInt16+ | UInt32+ | UInt64+ | Float16+ | Float32+ | Float64+ | Decimal32(_, _)+ | Decimal64(_, _)+ | Decimal128(_, _)+ | Decimal256(_, _)+ | Date32+ | Date64+ | Time32(_)+ | Time64(_)+ | Timestamp(_, _)+ | Utf8+ | LargeUtf8+ | Utf8View+ | Binary+ | LargeBinary+ | BinaryView+ )+}+#[user_doc(doc_section(label = "General Functions"),description = "Returns the first element in an aggregation group according to the requested ordering. If no ordering is given, returns an arbitrary element from the group.",@@ -178,110 +315,14 @@ impl AggregateUDFImpl for FirstValue {}fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {- use DataType::*;- !args.order_bys.is_empty()- && matches!(- args.return_field.data_type(),- Int8 | Int16- | Int32- | Int64- | UInt8- | UInt16- | UInt32- | UInt64- | Float16- | Float32- | Float64- | Decimal32(_, _)- | Decimal64(_, _)- | Decimal128(_, _)- | Decimal256(_, _)- | Date32- | Date64- | Time32(_)- | Time64(_)- | Timestamp(_, _)- )+ groups_accumulator_supported(&args)}fn create_groups_accumulator(&self,args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {- fn create_accumulator<T: ArrowPrimitiveType + Send>(- args: &AccumulatorArgs,- ) -> Result<Box<dyn GroupsAccumulator>> {- let Some(ordering) = LexOrdering::new(args.order_bys.to_vec()) else {- return internal_err!("Groups accumulator must have an ordering.");- };-- let ordering_dtypes = ordering- .iter()- .map(|e| e.expr.data_type(args.schema))- .collect::<Result<Vec<_>>>()?;-- FirstPrimitiveGroupsAccumulator::<T>::try_new(- ordering,- args.ignore_nulls,- args.return_field.data_type(),- &ordering_dtypes,- true,- )- .map(|acc| Box::new(acc) as _)- }-- match args.return_field.data_type() {- DataType::Int8 => create_accumulator::<Int8Type>(&args),- DataType::Int16 => create_accumulator::<Int16Type>(&args),- DataType::Int32 => create_accumulator::<Int32Type>(&args),- DataType::Int64 => create_accumulator::<Int64Type>(&args),- DataType::UInt8 => create_accumulator::<UInt8Type>(&args),- DataType::UInt16 => create_accumulator::<UInt16Type>(&args),- DataType::UInt32 => create_accumulator::<UInt32Type>(&args),- DataType::UInt64 => create_accumulator::<UInt64Type>(&args),- DataType::Float16 => create_accumulator::<Float16Type>(&args),- DataType::Float32 => create_accumulator::<Float32Type>(&args),- DataType::Float64 => create_accumulator::<Float64Type>(&args),-- DataType::Decimal32(_, _) => create_accumulator::<Decimal32Type>(&args),- DataType::Decimal64(_, _) => create_accumulator::<Decimal64Type>(&args),- DataType::Decimal128(_, _) => create_accumulator::<Decimal128Type>(&args),- DataType::Decimal256(_, _) => create_accumulator::<Decimal256Type>(&args),-- DataType::Timestamp(TimeUnit::Second, _) => {- create_accumulator::<TimestampSecondType>(&args)- }- DataType::Timestamp(TimeUnit::Millisecond, _) => {- create_accumulator::<TimestampMillisecondType>(&args)- }- DataType::Timestamp(TimeUnit::Microsecond, _) => {- create_accumulator::<TimestampMicrosecondType>(&args)- }- DataType::Timestamp(TimeUnit::Nanosecond, _) => {- create_accumulator::<TimestampNanosecondType>(&args)- }-- DataType::Date32 => create_accumulator::<Date32Type>(&args),- DataType::Date64 => create_accumulator::<Date64Type>(&args),- DataType::Time32(TimeUnit::Second) => {- create_accumulator::<Time32SecondType>(&args)- }- DataType::Time32(TimeUnit::Millisecond) => {- create_accumulator::<Time32MillisecondType>(&args)- }-- DataType::Time64(TimeUnit::Microsecond) => {- create_accumulator::<Time64MicrosecondType>(&args)- }- DataType::Time64(TimeUnit::Nanosecond) => {- create_accumulator::<Time64NanosecondType>(&args)- }-- _ => internal_err!(- "GroupsAccumulator not supported for first_value({})",- args.return_field.data_type()- ),- }+ create_groups_accumulator(&args, true, self.name())}fn with_beneficial_ordering(@@ -311,13 +352,9 @@ impl AggregateUDFImpl for FirstValue {}}-// TODO: rename to PrimitiveGroupsAccumulator-struct FirstPrimitiveGroupsAccumulator<T>-where- T: ArrowPrimitiveType + Send,-{+struct FirstLastGroupsAccumulator<S: ValueState> {// ================ state ===========- vals: Vec<T::Native>,+ state: S,// Stores ordering values, of the aggregator requirement corresponding to first value// of the aggregator.// The `orderings` are stored row-wise, meaning that `orderings[group_idx]`@@ -326,19 +363,16 @@ where// At the beginning, `is_sets[group_idx]` is false, which means `first` is not seen yet.// Once we see the first value, we set the `is_sets[group_idx]` flagis_sets: BooleanBufferBuilder,- // null_builder[group_idx] == false => vals[group_idx] is null- null_builder: BooleanBufferBuilder,// size of `self.orderings`// Calculating the memory usage of `self.orderings` using `ScalarValue::size_of_vec` is quite costly.// Therefore, we cache it and compute `size_of` only after each update// to avoid calling `ScalarValue::size_of_vec` by Self.size.size_of_orderings: usize,- // buffer for `get_filtered_min_of_each_group`+ // buffer for `get_filtered_extreme_of_each_group`// filter_min_of_each_group_buf.0[group_idx] -> idx_in_val// only valid if filter_min_of_each_group_buf.1[group_idx] == true- // TODO: rename to extreme_of_each_group_buf- min_of_each_group_buf: (Vec<usize>, BooleanBufferBuilder),+ extreme_of_each_group_buf: (Vec<usize>, BooleanBufferBuilder),// =========== option ============@@ -351,19 +385,14 @@ wheresort_options: Vec<SortOptions>,// Ignore null values.ignore_nulls: bool,- /// The output type- data_type: DataType,default_orderings: Vec<ScalarValue>,}-impl<T> FirstPrimitiveGroupsAccumulator<T>-where- T: ArrowPrimitiveType + Send,-{+impl<S: ValueState> FirstLastGroupsAccumulator<S> {fn try_new(+ state: S,ordering_req: LexOrdering,ignore_nulls: bool,- data_type: &DataType,ordering_dtypes: &[DataType],pick_first_in_group: bool,) -> Result<Self> {@@ -375,17 +404,15 @@ wherelet sort_options = get_sort_options(&ordering_req);Ok(Self {- null_builder: BooleanBufferBuilder::new(0),ordering_req,sort_options,ignore_nulls,default_orderings,- data_type: data_type.clone(),- vals: Vec::new(),+ state,orderings: Vec::new(),is_sets: BooleanBufferBuilder::new(0),size_of_orderings: 0,- min_of_each_group_buf: (Vec::new(), BooleanBufferBuilder::new(0)),+ extreme_of_each_group_buf: (Vec::new(), BooleanBufferBuilder::new(0)),pick_first_in_group,})}@@ -424,32 +451,8 @@ whereresult}- fn take_need(- bool_buf_builder: &mut BooleanBufferBuilder,- emit_to: EmitTo,- ) -> BooleanBuffer {- let bool_buf = bool_buf_builder.finish();- match emit_to {- EmitTo::All => bool_buf,- EmitTo::First(n) => {- // split off the first N values in seen_values- //- // TODO make this more efficient rather than two- // copies and bitwise manipulation- let first_n: BooleanBuffer = bool_buf.iter().take(n).collect();- // reset the existing buffer- for b in bool_buf.iter().skip(n) {- bool_buf_builder.append(b);- }- first_n- }- }- }-fn resize_states(&mut self, new_size: usize) {- self.vals.resize(new_size, T::default_value());-- self.null_builder.resize(new_size);+ self.state.resize(new_size);if self.orderings.len() < new_size {let current_len = self.orderings.len();@@ -468,44 +471,43 @@ whereself.is_sets.resize(new_size);- self.min_of_each_group_buf.0.resize(new_size, 0);- self.min_of_each_group_buf.1.resize(new_size);+ self.extreme_of_each_group_buf.0.resize(new_size, 0);+ self.extreme_of_each_group_buf.1.resize(new_size);}fn update_state(&mut self,group_idx: usize,orderings: &[ScalarValue],- new_val: T::Native,- is_null: bool,- ) {- self.vals[group_idx] = new_val;+ array: &ArrayRef,+ idx: usize,+ ) -> Result<()> {+ self.state.update(group_idx, array, idx)?;self.is_sets.set_bit(group_idx, true);- self.null_builder.set_bit(group_idx, !is_null);-assert!(orderings.len() == self.ordering_req.len());let old_size = ScalarValue::size_of_vec(&self.orderings[group_idx]);self.orderings[group_idx].clear();self.orderings[group_idx].extend_from_slice(orderings);let new_size = ScalarValue::size_of_vec(&self.orderings[group_idx]);self.size_of_orderings = self.size_of_orderings - old_size + new_size;+ Ok(())}fn take_state(&mut self,emit_to: EmitTo,- ) -> (ArrayRef, Vec<Vec<ScalarValue>>, BooleanBuffer) {- emit_to.take_needed(&mut self.min_of_each_group_buf.0);- self.min_of_each_group_buf+ ) -> Result<(ArrayRef, Vec<Vec<ScalarValue>>, BooleanBuffer)> {+ emit_to.take_needed(&mut self.extreme_of_each_group_buf.0);+ self.extreme_of_each_group_buf.1- .truncate(self.min_of_each_group_buf.0.len());+ .truncate(self.extreme_of_each_group_buf.0.len());- (- self.take_vals_and_null_buf(emit_to),+ Ok((+ self.state.take(emit_to)?,self.take_orderings(emit_to),- Self::take_need(&mut self.is_sets, emit_to),- )+ state::take_need(&mut self.is_sets, emit_to),+ ))}// should be used in test only@@ -519,20 +521,19 @@ where/// Returns a vector of tuples `(group_idx, idx_in_val)` representing the index of the/// minimum value in `orderings` for each group, using lexicographical comparison./// Values are filtered using `opt_filter` and `is_set_arr` if provided.- /// TODO: rename to get_filtered_extreme_of_each_group- fn get_filtered_min_of_each_group(+ fn get_filtered_extreme_of_each_group(&mut self,orderings: &[ArrayRef],group_indices: &[usize],opt_filter: Option<&BooleanArray>,- vals: &PrimitiveArray<T>,+ vals: &ArrayRef,is_set_arr: Option<&BooleanArray>,) -> Result<Vec<(usize, usize)>> {// Set all values in min_of_each_group_buf.1 to false.- self.min_of_each_group_buf.1.truncate(0);- self.min_of_each_group_buf+ self.extreme_of_each_group_buf.1.truncate(0);+ self.extreme_of_each_group_buf.1- .append_n(self.vals.len(), false);+ .append_n(self.is_sets.len(), false);// No need to call `clear` since `self.min_of_each_group_buf.0[group_idx]`// is only valid when `self.min_of_each_group_buf.1[group_idx] == true`.@@ -565,48 +566,35 @@ wherecontinue;}- let is_valid = self.min_of_each_group_buf.1.get_bit(group_idx);+ let is_valid = self.extreme_of_each_group_buf.1.get_bit(group_idx);if !is_valid {- self.min_of_each_group_buf.1.set_bit(group_idx, true);- self.min_of_each_group_buf.0[group_idx] = idx_in_val;+ self.extreme_of_each_group_buf.1.set_bit(group_idx, true);+ self.extreme_of_each_group_buf.0[group_idx] = idx_in_val;} else {let ordering = comparator- .compare(self.min_of_each_group_buf.0[group_idx], idx_in_val);+ .compare(self.extreme_of_each_group_buf.0[group_idx], idx_in_val);if (ordering.is_gt() && self.pick_first_in_group)|| (ordering.is_lt() && !self.pick_first_in_group){- self.min_of_each_group_buf.0[group_idx] = idx_in_val;+ self.extreme_of_each_group_buf.0[group_idx] = idx_in_val;}}}Ok(self- .min_of_each_group_buf+ .extreme_of_each_group_buf.0.iter().enumerate()- .filter(|(group_idx, _)| self.min_of_each_group_buf.1.get_bit(*group_idx))+ .filter(|(group_idx, _)| self.extreme_of_each_group_buf.1.get_bit(*group_idx)).map(|(group_idx, idx_in_val)| (group_idx, *idx_in_val)).collect::<Vec<_>>())}-- fn take_vals_and_null_buf(&mut self, emit_to: EmitTo) -> ArrayRef {- let r = emit_to.take_needed(&mut self.vals);-- let null_buf = NullBuffer::new(Self::take_need(&mut self.null_builder, emit_to));-- let values = PrimitiveArray::<T>::new(r.into(), Some(null_buf)) // no copy- .with_data_type(self.data_type.clone());- Arc::new(values)- }}-impl<T> GroupsAccumulator for FirstPrimitiveGroupsAccumulator<T>-where- T: ArrowPrimitiveType + Send,-{+impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S> {fn update_batch(&mut self,// e.g. first_value(a order by b): values_and_order_cols will be [a, b]@@ -617,13 +605,13 @@ where) -> Result<()> {self.resize_states(total_num_groups);- let vals = values_and_order_cols[0].as_primitive::<T>();+ let vals = &values_and_order_cols[0];let mut ordering_buf = Vec::with_capacity(self.ordering_req.len());// The overhead of calling `extract_row_at_idx_to_buf` is somewhat high, so we need to minimize its calls as much as possible.for (group_idx, idx) in self- .get_filtered_min_of_each_group(+ .get_filtered_extreme_of_each_group(&values_and_order_cols[1..],group_indices,opt_filter,@@ -639,12 +627,7 @@ where)?;if self.should_update_state(group_idx, &ordering_buf)? {- self.update_state(- group_idx,- &ordering_buf,- vals.value(idx),- vals.is_null(idx),- );+ self.update_state(group_idx, &ordering_buf, vals, idx)?;}}@@ -652,11 +635,11 @@ where}fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {- Ok(self.take_state(emit_to).0)+ Ok(self.take_state(emit_to)?.0)}fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {- let (val_arr, orderings, is_sets) = self.take_state(emit_to);+ let (val_arr, orderings, is_sets) = self.take_state(emit_to)?;let mut result = Vec::with_capacity(self.orderings.len() + 2);result.push(val_arr);@@ -702,9 +685,9 @@ wherelet is_set_arr = as_boolean_array(is_set_arr)?;- let vals = values[0].as_primitive::<T>();+ let vals = &values[0];// The overhead of calling `extract_row_at_idx_to_buf` is somewhat high, so we need to minimize its calls as much as possible.- let groups = self.get_filtered_min_of_each_group(+ let groups = self.get_filtered_extreme_of_each_group(&val_and_order_cols[1..],group_indices,opt_filter,@@ -716,12 +699,7 @@ whereextract_row_at_idx_to_buf(&val_and_order_cols[1..], idx, &mut ordering_buf)?;if self.should_update_state(group_idx, &ordering_buf)? {- self.update_state(- group_idx,- &ordering_buf,- vals.value(idx),- vals.is_null(idx),- );+ self.update_state(group_idx, &ordering_buf, vals, idx)?;}}@@ -729,12 +707,11 @@ where}fn size(&self) -> usize {- self.vals.capacity() * size_of::<T::Native>()- + self.null_builder.capacity() / 8 // capacity is in bits, so convert to bytes- + self.is_sets.capacity() / 8+ self.state.size()+ + self.is_sets.capacity() / 8 // capacity is in bits, so convert to bytes+ self.size_of_orderings- + self.min_of_each_group_buf.0.capacity() * size_of::<usize>()- + self.min_of_each_group_buf.1.capacity() / 8+ + self.extreme_of_each_group_buf.0.capacity() * size_of::<usize>()+ + self.extreme_of_each_group_buf.1.capacity() / 8}fn supports_convert_to_state(&self) -> bool {@@ -1140,114 +1117,14 @@ impl AggregateUDFImpl for LastValue {}fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {- use DataType::*;- !args.order_bys.is_empty()- && matches!(- args.return_field.data_type(),- Int8 | Int16- | Int32- | Int64- | UInt8- | UInt16- | UInt32- | UInt64- | Float16- | Float32- | Float64- | Decimal32(_, _)- | Decimal64(_, _)- | Decimal128(_, _)- | Decimal256(_, _)- | Date32- | Date64- | Time32(_)- | Time64(_)- | Timestamp(_, _)- )+ groups_accumulator_supported(&args)}fn create_groups_accumulator(&self,args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {- fn create_accumulator<T>(- args: &AccumulatorArgs,- ) -> Result<Box<dyn GroupsAccumulator>>- where- T: ArrowPrimitiveType + Send,- {- let Some(ordering) = LexOrdering::new(args.order_bys.to_vec()) else {- return internal_err!("Groups accumulator must have an ordering.");- };-- let ordering_dtypes = ordering- .iter()- .map(|e| e.expr.data_type(args.schema))- .collect::<Result<Vec<_>>>()?;-- Ok(Box::new(FirstPrimitiveGroupsAccumulator::<T>::try_new(- ordering,- args.ignore_nulls,- args.return_field.data_type(),- &ordering_dtypes,- false,- )?))- }-- match args.return_field.data_type() {- DataType::Int8 => create_accumulator::<Int8Type>(&args),- DataType::Int16 => create_accumulator::<Int16Type>(&args),- DataType::Int32 => create_accumulator::<Int32Type>(&args),- DataType::Int64 => create_accumulator::<Int64Type>(&args),- DataType::UInt8 => create_accumulator::<UInt8Type>(&args),- DataType::UInt16 => create_accumulator::<UInt16Type>(&args),- DataType::UInt32 => create_accumulator::<UInt32Type>(&args),- DataType::UInt64 => create_accumulator::<UInt64Type>(&args),- DataType::Float16 => create_accumulator::<Float16Type>(&args),- DataType::Float32 => create_accumulator::<Float32Type>(&args),- DataType::Float64 => create_accumulator::<Float64Type>(&args),-- DataType::Decimal32(_, _) => create_accumulator::<Decimal32Type>(&args),- DataType::Decimal64(_, _) => create_accumulator::<Decimal64Type>(&args),- DataType::Decimal128(_, _) => create_accumulator::<Decimal128Type>(&args),- DataType::Decimal256(_, _) => create_accumulator::<Decimal256Type>(&args),-- DataType::Timestamp(TimeUnit::Second, _) => {- create_accumulator::<TimestampSecondType>(&args)- }- DataType::Timestamp(TimeUnit::Millisecond, _) => {- create_accumulator::<TimestampMillisecondType>(&args)- }- DataType::Timestamp(TimeUnit::Microsecond, _) => {- create_accumulator::<TimestampMicrosecondType>(&args)- }- DataType::Timestamp(TimeUnit::Nanosecond, _) => {- create_accumulator::<TimestampNanosecondType>(&args)- }-- DataType::Date32 => create_accumulator::<Date32Type>(&args),- DataType::Date64 => create_accumulator::<Date64Type>(&args),- DataType::Time32(TimeUnit::Second) => {- create_accumulator::<Time32SecondType>(&args)- }- DataType::Time32(TimeUnit::Millisecond) => {- create_accumulator::<Time32MillisecondType>(&args)- }-- DataType::Time64(TimeUnit::Microsecond) => {- create_accumulator::<Time64MicrosecondType>(&args)- }- DataType::Time64(TimeUnit::Nanosecond) => {- create_accumulator::<Time64NanosecondType>(&args)- }-- _ => {- internal_err!(- "GroupsAccumulator not supported for last_value({})",- args.return_field.data_type()- )- }- }+ create_groups_accumulator(&args, false, self.name())}}@@ -1541,7 +1418,7 @@ mod tests {use std::iter::repeat_with;use arrow::{- array::{BooleanArray, Int64Array, ListArray, StringArray},+ array::{BooleanArray, Int64Array, ListArray, PrimitiveArray, StringArray},compute::SortOptions,datatypes::Schema,};@@ -1668,10 +1545,10 @@ mod tests {options: SortOptions::default(),}];- let mut group_acc = FirstPrimitiveGroupsAccumulator::<Int64Type>::try_new(+ let mut group_acc = FirstLastGroupsAccumulator::try_new(+ PrimitiveValueState::<Int64Type>::new(DataType::Int64),sort_keys.into(),true,- &DataType::Int64,&[DataType::Int64],true,)?;@@ -1762,10 +1639,10 @@ mod tests {options: SortOptions::default(),}];- let mut group_acc = FirstPrimitiveGroupsAccumulator::<Int64Type>::try_new(+ let mut group_acc = FirstLastGroupsAccumulator::try_new(+ PrimitiveValueState::<Int64Type>::new(DataType::Int64),sort_keys.into(),true,- &DataType::Int64,&[DataType::Int64],true,)?;@@ -1843,10 +1720,10 @@ mod tests {options: SortOptions::default(),}];- let mut group_acc = FirstPrimitiveGroupsAccumulator::<Int64Type>::try_new(+ let mut group_acc = FirstLastGroupsAccumulator::try_new(+ PrimitiveValueState::<Int64Type>::new(DataType::Int64),sort_keys.into(),true,- &DataType::Int64,&[DataType::Int64],false,)?;diff --git a/datafusion/functions-aggregate/src/first_last/state.rs b/datafusion/functions-aggregate/src/first_last/state.rsnew file mode 100644index 0000000000..3aa578c74c--- /dev/null+++ b/datafusion/functions-aggregate/src/first_last/state.rs@@ -0,0 +1,466 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::mem::size_of;+use std::sync::Arc;++use arrow::array::{+ Array, ArrayRef, ArrowPrimitiveType, AsArray, BinaryBuilder, BinaryViewBuilder,+ BooleanBufferBuilder, LargeBinaryBuilder, LargeStringBuilder, PrimitiveArray,+ StringBuilder, StringViewBuilder,+};+use arrow::buffer::{BooleanBuffer, NullBuffer};+use arrow::datatypes::DataType;+use datafusion_common::{Result, internal_err};+use datafusion_expr::EmitTo;++pub(crate) trait ValueState: Send + Sync {+ /// Resizes the state to accommodate `new_size` groups.+ fn resize(&mut self, new_size: usize);+ /// Updates the state for the specified `group_idx` using the value at `idx` from the provided `array`.+ ///+ /// Note: While this is not a batch interface, it is not a performance bottleneck.+ /// In heavy aggregation benchmarks, the overhead of this method is typically less than 1%.+ ///+ /// ```sql+ /// -- TPC-H SF10+ /// select l_shipmode, last_value(l_partkey order by l_orderkey, l_linenumber, l_comment, l_suppkey, l_tax)+ /// from 'benchmarks/data/tpch_sf10/lineitem'+ /// group by l_shipmode;+ ///+ /// -- H2O G1_1e8+ /// select t.id1, first_value(t.id3 order by t.id2, t.id4) as r2+ /// from 'benchmarks/data/h2o/G1_1e8_1e8_100_0.parquet' as t+ /// group by t.id1, t.v1;+ /// ```+ fn update(&mut self, group_idx: usize, array: &ArrayRef, idx: usize) -> Result<()>;+ /// Takes the accumulated state and returns it as an [`ArrayRef`], respecting the `emit_to` strategy.+ fn take(&mut self, emit_to: EmitTo) -> Result<ArrayRef>;+ /// Returns the estimated memory size of the state in bytes.+ fn size(&self) -> usize;+}++pub(crate) struct PrimitiveValueState<T: ArrowPrimitiveType + Send> {+ /// Values data+ vals: Vec<T::Native>,+ nulls: BooleanBufferBuilder,+ data_type: DataType,+}++impl<T: ArrowPrimitiveType + Send> PrimitiveValueState<T> {+ pub(crate) fn new(data_type: DataType) -> Self {+ Self {+ vals: vec![],+ nulls: BooleanBufferBuilder::new(0),+ data_type,+ }+ }+}++impl<T: ArrowPrimitiveType + Send> ValueState for PrimitiveValueState<T> {+ fn resize(&mut self, new_size: usize) {+ self.vals.resize(new_size, T::default_value());+ self.nulls.resize(new_size);+ }++ fn update(&mut self, group_idx: usize, array: &ArrayRef, idx: usize) -> Result<()> {+ let array = array.as_primitive::<T>();+ self.vals[group_idx] = array.value(idx);+ self.nulls.set_bit(group_idx, !array.is_null(idx));+ Ok(())+ }++ fn take(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {+ let values = emit_to.take_needed(&mut self.vals);+ let null_buf = NullBuffer::new(take_need(&mut self.nulls, emit_to));+ let array: PrimitiveArray<T> =+ PrimitiveArray::<T>::new(values.into(), Some(null_buf))+ .with_data_type(self.data_type.clone());+ Ok(Arc::new(array))+ }++ fn size(&self) -> usize {+ self.vals.capacity() * size_of::<T::Native>() + self.nulls.capacity() / 8+ }+}++/// Stores internal state for "bytes" types (Utf8, Binary, etc.).+///+/// This implementation is similar to `MinMaxBytesState` in `min_max_bytes.rs`, but+/// it does not reuse it for two main reasons:+///+/// 1. **Direct Overwrite**: `MinMaxBytesState::update_batch` is tightly coupled+/// with min/max comparison logic, whereas `FirstLast` performs its own comparisons+/// externally (using ordering columns) and only needs a simple interface to+/// unconditionally set/overwrite values for specific groups.+/// 2. **Different NULL Handling**: `MinMaxBytesState` always ignores `NULL` values+/// in the input, while `BytesValueState` needs to support setting `NULL` values+/// to correctly implement `RESPECT NULLS` behavior.+///+pub(crate) struct BytesValueState {+ vals: Vec<Option<Vec<u8>>>,+ data_type: DataType,+ /// The sum of the capacities of all vectors in `vals`.+ total_capacity: usize,+}++impl BytesValueState {+ pub(crate) fn try_new(data_type: DataType) -> Result<Self> {+ if !matches!(+ data_type,+ DataType::Utf8+ | DataType::LargeUtf8+ | DataType::Utf8View+ | DataType::Binary+ | DataType::LargeBinary+ | DataType::BinaryView+ ) {+ return internal_err!("BytesValueState does not support {}", data_type);+ }+ Ok(Self {+ vals: vec![],+ data_type,+ total_capacity: 0,+ })+ }+}++impl ValueState for BytesValueState {+ fn resize(&mut self, new_size: usize) {+ if new_size < self.vals.len() {+ for v in self.vals[new_size..].iter().flatten() {+ self.total_capacity -= v.capacity();+ }+ }+ self.vals.resize(new_size, None);+ }++ fn update(&mut self, group_idx: usize, array: &ArrayRef, idx: usize) -> Result<()> {+ if let Some(v) = &self.vals[group_idx] {+ self.total_capacity -= v.capacity();+ }++ if array.is_null(idx) {+ self.vals[group_idx] = None;+ } else {+ let val = match self.data_type {+ DataType::Utf8 => array.as_string::<i32>().value(idx).as_bytes(),+ DataType::LargeUtf8 => array.as_string::<i64>().value(idx).as_bytes(),+ DataType::Utf8View => array.as_string_view().value(idx).as_bytes(),+ DataType::Binary => array.as_binary::<i32>().value(idx),+ DataType::LargeBinary => array.as_binary::<i64>().value(idx),+ DataType::BinaryView => array.as_binary_view().value(idx),+ _ => {+ return internal_err!(+ "Unsupported data type for BytesValueState: {}",+ self.data_type+ );+ }+ };++ if let Some(v) = &mut self.vals[group_idx] {+ v.clear();+ v.extend_from_slice(val);+ } else {+ let v = val.to_vec();+ self.vals[group_idx] = Some(v);+ }++ self.vals[group_idx]+ .as_ref()+ .inspect(|x| self.total_capacity += x.capacity());+ }+ Ok(())+ }++ fn take(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {+ let values = emit_to.take_needed(&mut self.vals);++ let (total_len, taken_capacity) = values+ .iter()+ .flatten()+ .fold((0, 0), |(len_acc, cap_acc), v| {+ (len_acc + v.len(), cap_acc + v.capacity())+ });+ self.total_capacity -= taken_capacity;++ match self.data_type {+ DataType::Utf8 => {+ let mut builder = StringBuilder::with_capacity(values.len(), total_len);+ for val in values {+ match val {+ Some(v) => builder.append_value(+ // SAFETY: The bytes were originally from a valid UTF-8 array in `update`+ unsafe { std::str::from_utf8_unchecked(&v) },+ ),+ None => builder.append_null(),+ }+ }+ Ok(Arc::new(builder.finish()))+ }+ DataType::LargeUtf8 => {+ let mut builder =+ LargeStringBuilder::with_capacity(values.len(), total_len);+ for val in values {+ match val {+ Some(v) => builder.append_value(+ // SAFETY: The bytes were originally from a valid UTF-8 array in `update`+ unsafe { std::str::from_utf8_unchecked(&v) },+ ),+ None => builder.append_null(),+ }+ }+ Ok(Arc::new(builder.finish()))+ }+ DataType::Utf8View => {+ let mut builder = StringViewBuilder::with_capacity(values.len());+ for val in values {+ match val {+ Some(v) => builder.append_value(+ // SAFETY: The bytes were originally from a valid UTF-8 array in `update`+ unsafe { std::str::from_utf8_unchecked(&v) },+ ),+ None => builder.append_null(),+ }+ }+ Ok(Arc::new(builder.finish()))+ }+ DataType::Binary => {+ let mut builder = BinaryBuilder::with_capacity(values.len(), total_len);+ for val in values {+ match val {+ Some(v) => builder.append_value(&v),+ None => builder.append_null(),+ }+ }+ Ok(Arc::new(builder.finish()))+ }+ DataType::LargeBinary => {+ let mut builder =+ LargeBinaryBuilder::with_capacity(values.len(), total_len);+ for val in values {+ match val {+ Some(v) => builder.append_value(&v),+ None => builder.append_null(),+ }+ }+ Ok(Arc::new(builder.finish()))+ }+ DataType::BinaryView => {+ let mut builder = BinaryViewBuilder::with_capacity(values.len());+ for val in values {+ match val {+ Some(v) => builder.append_value(&v),+ None => builder.append_null(),+ }+ }+ Ok(Arc::new(builder.finish()))+ }+ _ => internal_err!(+ "Unsupported data type for BytesValueState: {}",+ self.data_type+ ),+ }+ }++ fn size(&self) -> usize {+ self.vals.capacity() * size_of::<Option<Vec<u8>>>() + self.total_capacity+ }+}++impl BytesValueState {+ #[cfg(test)]+ /// For testing only: strictly calculate the sum of capacities of all vectors in `vals`.+ fn total_capacity_calculated(&self) -> usize {+ self.vals.iter().flatten().map(|v| v.capacity()).sum()+ }+}++pub(crate) fn take_need(+ bool_buf_builder: &mut BooleanBufferBuilder,+ emit_to: EmitTo,+) -> BooleanBuffer {+ let bool_buf = bool_buf_builder.finish();+ match emit_to {+ EmitTo::All => bool_buf,+ EmitTo::First(n) => {+ // split off the first N values in seen_values+ //+ // TODO make this more efficient rather than two+ // copies and bitwise manipulation+ let first_n: BooleanBuffer = bool_buf.iter().take(n).collect();+ // reset the existing buffer+ for b in bool_buf.iter().skip(n) {+ bool_buf_builder.append(b);+ }+ first_n+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::{+ BinaryArray, BinaryViewArray, LargeBinaryArray, LargeStringArray, StringArray,+ StringViewArray,+ };++ #[test]+ fn test_bytes_value_state_utf8() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::Utf8)?;+ state.resize(2);++ let array: ArrayRef = Arc::new(StringArray::from(vec![+ Some("hello"),+ Some("world"),+ Some("longer_string_than_hello"),+ ]));++ state.update(0, &array, 0)?; // group 0 = "hello"+ state.update(1, &array, 1)?; // group 1 = "world"++ assert_eq!(state.total_capacity, state.total_capacity_calculated());++ // Overwrite group 0 with a longer string (checks capacity update logic)+ state.update(0, &array, 2)?;+ assert_eq!(state.total_capacity, state.total_capacity_calculated());++ let result = state.take(EmitTo::All)?;+ let result = result.as_string::<i32>();+ assert_eq!(result.len(), 2);+ assert_eq!(result.value(0), "longer_string_than_hello");+ assert_eq!(result.value(1), "world");++ // After take all, size should be 0 (excluding vals vector capacity)+ assert_eq!(state.total_capacity, 0);+ assert_eq!(state.total_capacity, state.total_capacity_calculated());++ Ok(())+ }++ #[test]+ fn test_bytes_value_state_large_utf8() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::LargeUtf8)?;+ state.resize(1);+ let array: ArrayRef = Arc::new(LargeStringArray::from(vec!["large_utf8"]));+ state.update(0, &array, 0)?;+ let result = state.take(EmitTo::All)?;+ assert_eq!(result.as_string::<i64>().value(0), "large_utf8");+ Ok(())+ }++ #[test]+ fn test_bytes_value_state_utf8_view() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::Utf8View)?;+ state.resize(1);+ let array: ArrayRef = Arc::new(StringViewArray::from(vec!["Utf8View"]));+ state.update(0, &array, 0)?;+ let result = state.take(EmitTo::All)?;+ assert_eq!(result.as_string_view().value(0), "Utf8View");+ Ok(())+ }++ #[test]+ fn test_bytes_value_state_binary() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::Binary)?;+ state.resize(1);+ let array: ArrayRef = Arc::new(BinaryArray::from(vec![b"binary" as &[u8]]));+ state.update(0, &array, 0)?;+ let result = state.take(EmitTo::All)?;+ assert_eq!(result.as_binary::<i32>().value(0), b"binary");+ Ok(())+ }++ #[test]+ fn test_bytes_value_state_large_binary() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::LargeBinary)?;+ state.resize(1);+ let array: ArrayRef =+ Arc::new(LargeBinaryArray::from(vec![b"large_binary" as &[u8]]));+ state.update(0, &array, 0)?;+ let result = state.take(EmitTo::All)?;+ assert_eq!(result.as_binary::<i64>().value(0), b"large_binary");+ Ok(())+ }++ #[test]+ fn test_bytes_value_state_binary_view() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::BinaryView)?;+ state.resize(1);++ let data: Vec<Option<&[u8]>> = vec![Some(b"long_binary_value_to_test_view")];+ let array: ArrayRef = Arc::new(BinaryViewArray::from(data));++ state.update(0, &array, 0)?;++ let result = state.take(EmitTo::All)?;+ let result = result.as_binary_view();+ assert_eq!(result.value(0), b"long_binary_value_to_test_view");++ Ok(())+ }++ #[test]+ fn test_bytes_value_state_emit_first() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::Utf8)?;+ state.resize(3);++ let array: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"]));+ state.update(0, &array, 0)?;+ state.update(1, &array, 1)?;+ state.update(2, &array, 2)?;++ let result = state.take(EmitTo::First(2))?;+ let result = result.as_string::<i32>();+ assert_eq!(result.len(), 2);+ assert_eq!(result.value(0), "a");+ assert_eq!(result.value(1), "b");++ // Remaining should be "c"+ let result = state.take(EmitTo::All)?;+ let result = result.as_string::<i32>();+ assert_eq!(result.len(), 1);+ assert_eq!(result.value(0), "c");++ Ok(())+ }++ #[test]+ fn test_bytes_value_state_update_null() -> Result<()> {+ let mut state = BytesValueState::try_new(DataType::Utf8)?;+ state.resize(1);++ let array: ArrayRef = Arc::new(StringArray::from(vec![Some("hello"), None]));++ // group 0 = "hello"+ state.update(0, &array, 0)?;+ assert_eq!(state.total_capacity, state.total_capacity_calculated());+ assert!(state.total_capacity > 0);++ // group 0 = NULL+ state.update(0, &array, 1)?;+ assert_eq!(+ state.total_capacity,+ state.total_capacity_calculated(),+ "total_capacity should match calculated capacity after update(NULL)"+ );+ assert_eq!(state.total_capacity, 0);++ Ok(())+ }+}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 4eb33f0cb5..0232367163 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -8817,3 +8817,121 @@ set datafusion.execution.target_partitions = 4;statement okDROP TABLE stream_test;+++#################+# first_value on strings/binary with groups and ordering+#################++statement ok+CREATE TABLE first_last_value_str_tests(id INT, sort_key INT, val TEXT) AS VALUES+(1, 10, 'apple'),+(1, 2, 'banana'),+(1, 5, 'cherry'),+(2, 100, 'dog'),+(2, 200, 'elephant');++# Utf8+query IT+SELECT id, first_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'Utf8') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 banana+2 dog++query IT+SELECT id, last_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'Utf8') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 apple+2 elephant++# LargeUtf8+query IT+SELECT id, first_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'LargeUtf8') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 banana+2 dog++query IT+SELECT id, last_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'LargeUtf8') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 apple+2 elephant++# Utf8View+query IT+SELECT id, first_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'Utf8View') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 banana+2 dog++query IT+SELECT id, last_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'Utf8View') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 apple+2 elephant++# Binary+query I?+SELECT id, first_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'Binary') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 62616e616e61+2 646f67++query I?+SELECT id, last_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'Binary') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 6170706c65+2 656c657068616e74++# LargeBinary+query I?+SELECT id, first_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'LargeBinary') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 62616e616e61+2 646f67++query I?+SELECT id, last_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(val, 'LargeBinary') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 6170706c65+2 656c657068616e74++# BinaryView+query I?+SELECT id, first_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(arrow_cast(val, 'Binary'), 'BinaryView') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 62616e616e61+2 646f67++query I?+SELECT id, last_value(val ORDER BY sort_key)+FROM (SELECT id, sort_key, arrow_cast(arrow_cast(val, 'Binary'), 'BinaryView') as val FROM first_last_value_str_tests)+GROUP BY id ORDER BY id;+----+1 6170706c65+2 656c657068616e74++statement ok+DROP TABLE first_last_value_str_tests;