Skip to content

Apache DataFusion SQL Query Engine

Introduce `expr_fields` to `AccumulatorArgs` to hold input argument fields (#18100)

Jeffrey Vo930620a919af
diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs
index 982b480459..62e8ab18b9 100644
--- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs
+++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs
@@ -954,13 +954,7 @@ impl AggregateUDFImpl for MetadataBasedAggregateUdf {
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
- let input_expr = acc_args
- .exprs
- .first()
- .ok_or(exec_datafusion_err!("Expected one argument"))?;
- let input_field = input_expr.return_field(acc_args.schema)?;
-
- let double_output = input_field
+ let double_output = acc_args.expr_fields[0]
.metadata()
.get("modify_values")
.map(|v| v == "double_output")
diff --git a/datafusion/ffi/src/udaf/accumulator_args.rs b/datafusion/ffi/src/udaf/accumulator_args.rs
index 0302c26a2e..6ac0a0b21d 100644
--- a/datafusion/ffi/src/udaf/accumulator_args.rs
+++ b/datafusion/ffi/src/udaf/accumulator_args.rs
@@ -97,6 +97,7 @@ impl TryFrom<AccumulatorArgs<'_>> for FFI_AccumulatorArgs {
pub struct ForeignAccumulatorArgs {
pub return_field: FieldRef,
pub schema: Schema,
+ pub expr_fields: Vec<FieldRef>,
pub ignore_nulls: bool,
pub order_bys: Vec<PhysicalSortExpr>,
pub is_reversed: bool,
@@ -132,9 +133,15 @@ impl TryFrom<FFI_AccumulatorArgs> for ForeignAccumulatorArgs {
let exprs = parse_physical_exprs(&proto_def.expr, &task_ctx, &schema, &codex)?;
+ let expr_fields = exprs
+ .iter()
+ .map(|e| e.return_field(&schema))
+ .collect::<Result<Vec<_>, _>>()?;
+
Ok(Self {
return_field,
schema,
+ expr_fields,
ignore_nulls: proto_def.ignore_nulls,
order_bys,
is_reversed: value.is_reversed,
@@ -150,6 +157,7 @@ impl<'a> From<&'a ForeignAccumulatorArgs> for AccumulatorArgs<'a> {
Self {
return_field: Arc::clone(&value.return_field),
schema: &value.schema,
+ expr_fields: &value.expr_fields,
ignore_nulls: value.ignore_nulls,
order_bys: &value.order_bys,
is_reversed: value.is_reversed,
@@ -175,6 +183,7 @@ mod tests {
let orig_args = AccumulatorArgs {
return_field: Field::new("f", DataType::Float64, true).into(),
schema: &schema,
+ expr_fields: &[Field::new("a", DataType::Int32, true).into()],
ignore_nulls: false,
order_bys: &[PhysicalSortExpr::new_default(col("a", &schema)?)],
is_reversed: false,
diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs
index 1ea1798c7c..ce5611590b 100644
--- a/datafusion/ffi/src/udaf/mod.rs
+++ b/datafusion/ffi/src/udaf/mod.rs
@@ -705,6 +705,7 @@ mod tests {
let acc_args = AccumulatorArgs {
return_field: Field::new("f", DataType::Float64, true).into(),
schema: &schema,
+ expr_fields: &[Field::new("a", DataType::Float64, true).into()],
ignore_nulls: true,
order_bys: &[PhysicalSortExpr::new_default(col("a", &schema)?)],
is_reversed: false,
@@ -782,6 +783,7 @@ mod tests {
let acc_args = AccumulatorArgs {
return_field: Field::new("f", DataType::Float64, true).into(),
schema: &schema,
+ expr_fields: &[Field::new("a", DataType::Float64, true).into()],
ignore_nulls: true,
order_bys: &[PhysicalSortExpr::new_default(col("a", &schema)?)],
is_reversed: false,
diff --git a/datafusion/functions-aggregate-common/src/accumulator.rs b/datafusion/functions-aggregate-common/src/accumulator.rs
index e0f7af1fb3..8db0ab4133 100644
--- a/datafusion/functions-aggregate-common/src/accumulator.rs
+++ b/datafusion/functions-aggregate-common/src/accumulator.rs
@@ -30,7 +30,8 @@ pub struct AccumulatorArgs<'a> {
/// The return field of the aggregate function.
pub return_field: FieldRef,
- /// The schema of the input arguments
+ /// Input schema to the aggregate function. If you need to check data type, nullability
+ /// or metadata of input arguments then you should use `expr_fields` below instead.
pub schema: &'a Schema,
/// Whether to ignore nulls.
@@ -67,6 +68,9 @@ pub struct AccumulatorArgs<'a> {
/// The physical expression of arguments the aggregate function takes.
pub exprs: &'a [Arc<dyn PhysicalExpr>],
+
+ /// Fields corresponding to each expr (same order & length).
+ pub expr_fields: &'a [FieldRef],
}
impl AccumulatorArgs<'_> {
diff --git a/datafusion/functions-aggregate/benches/count.rs b/datafusion/functions-aggregate/benches/count.rs
index 37c7fad4bd..2f42d66c7c 100644
--- a/datafusion/functions-aggregate/benches/count.rs
+++ b/datafusion/functions-aggregate/benches/count.rs
@@ -33,15 +33,17 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion};
fn prepare_group_accumulator() -> Box<dyn GroupsAccumulator> {
let schema = Arc::new(Schema::new(vec![Field::new("f", DataType::Int32, true)]));
+ let expr = col("f", &schema).unwrap();
let accumulator_args = AccumulatorArgs {
return_field: Field::new("f", DataType::Int64, true).into(),
schema: &schema,
+ expr_fields: &[expr.return_field(&schema).unwrap()],
ignore_nulls: false,
order_bys: &[],
is_reversed: false,
name: "COUNT(f)",
is_distinct: false,
- exprs: &[col("f", &schema).unwrap()],
+ exprs: &[expr],
};
let count_fn = Count::new();
@@ -56,15 +58,17 @@ fn prepare_accumulator() -> Box<dyn Accumulator> {
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
true,
)]));
+ let expr = col("f", &schema).unwrap();
let accumulator_args = AccumulatorArgs {
return_field: Arc::new(Field::new_list_field(DataType::Int64, true)),
schema: &schema,
+ expr_fields: &[expr.return_field(&schema).unwrap()],
ignore_nulls: false,
order_bys: &[],
is_reversed: false,
name: "COUNT(f)",
is_distinct: true,
- exprs: &[col("f", &schema).unwrap()],
+ exprs: &[expr],
};
let count_fn = Count::new();
diff --git a/datafusion/functions-aggregate/benches/min_max_bytes.rs b/datafusion/functions-aggregate/benches/min_max_bytes.rs
index a438ee5697..6d76ff2d03 100644
--- a/datafusion/functions-aggregate/benches/min_max_bytes.rs
+++ b/datafusion/functions-aggregate/benches/min_max_bytes.rs
@@ -44,6 +44,7 @@ fn create_max_bytes_accumulator() -> Box<dyn GroupsAccumulator> {
max.create_groups_accumulator(AccumulatorArgs {
return_field: Arc::new(Field::new("value", DataType::Utf8, true)),
schema: &input_schema,
+ expr_fields: &[Field::new("value", DataType::Utf8, true).into()],
ignore_nulls: true,
order_bys: &[],
is_reversed: false,
diff --git a/datafusion/functions-aggregate/benches/sum.rs b/datafusion/functions-aggregate/benches/sum.rs
index a1e9894fb8..6a21595927 100644
--- a/datafusion/functions-aggregate/benches/sum.rs
+++ b/datafusion/functions-aggregate/benches/sum.rs
@@ -31,8 +31,9 @@ fn prepare_accumulator(data_type: &DataType) -> Box<dyn GroupsAccumulator> {
let field = Field::new("f", data_type.clone(), true).into();
let schema = Arc::new(Schema::new(vec![Arc::clone(&field)]));
let accumulator_args = AccumulatorArgs {
- return_field: field,
+ return_field: Arc::clone(&field),
schema: &schema,
+ expr_fields: &[field],
ignore_nulls: false,
order_bys: &[],
is_reversed: false,
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index 9affdb3ee5..998f981dee 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -361,7 +361,7 @@ impl AggregateUDFImpl for ApproxDistinct {
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
- let data_type = acc_args.exprs[0].data_type(acc_args.schema)?;
+ let data_type = acc_args.expr_fields[0].data_type();
let accumulator: Box<dyn Accumulator> = match data_type {
// TODO u8, i8, u16, i16 shall really be done using bitmap, not HLL
diff --git a/datafusion/functions-aggregate/src/approx_median.rs b/datafusion/functions-aggregate/src/approx_median.rs
index 976f4d2c94..530dbf3e43 100644
--- a/datafusion/functions-aggregate/src/approx_median.rs
+++ b/datafusion/functions-aggregate/src/approx_median.rs
@@ -134,7 +134,7 @@ impl AggregateUDFImpl for ApproxMedian {
Ok(Box::new(ApproxPercentileAccumulator::new(
0.5_f64,
- acc_args.exprs[0].data_type(acc_args.schema)?,
+ acc_args.expr_fields[0].data_type().clone(),
)))
}
diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs b/datafusion/functions-aggregate/src/approx_percentile_cont.rs
index 668280314e..6513504b30 100644
--- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs
+++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs
@@ -187,9 +187,9 @@ impl ApproxPercentileCont {
None
};
- let data_type = args.exprs[0].data_type(args.schema)?;
+ let data_type = args.expr_fields[0].data_type();
let accumulator: ApproxPercentileAccumulator = match data_type {
- t @ (DataType::UInt8
+ DataType::UInt8
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
@@ -198,12 +198,11 @@ impl ApproxPercentileCont {
| DataType::Int32
| DataType::Int64
| DataType::Float32
- | DataType::Float64) => {
+ | DataType::Float64 => {
if let Some(max_size) = tdigest_max_size {
- ApproxPercentileAccumulator::new_with_max_size(percentile, t, max_size)
- }else{
- ApproxPercentileAccumulator::new(percentile, t)
-
+ ApproxPercentileAccumulator::new_with_max_size(percentile, data_type.clone(), max_size)
+ } else {
+ ApproxPercentileAccumulator::new(percentile, data_type.clone())
}
}
other => {
diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs b/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs
index 89ff546039..215341b507 100644
--- a/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs
+++ b/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs
@@ -220,7 +220,28 @@ impl AggregateUDFImpl for ApproxPercentileContWithWeight {
Arc::clone(&acc_args.exprs[2]), // percentile
]
},
- ..acc_args
+ expr_fields: if acc_args.exprs.len() == 4 {
+ &[
+ Arc::clone(&acc_args.expr_fields[0]), // value
+ Arc::clone(&acc_args.expr_fields[2]), // percentile
+ Arc::clone(&acc_args.expr_fields[3]), // centroids
+ ]
+ } else {
+ &[
+ Arc::clone(&acc_args.expr_fields[0]), // value
+ Arc::clone(&acc_args.expr_fields[2]), // percentile
+ ]
+ },
+ // Unchanged below; we list each field explicitly in case we ever add more
+ // fields to AccumulatorArgs making it easier to see if changes are also
+ // needed here.
+ return_field: acc_args.return_field,
+ schema: acc_args.schema,
+ ignore_nulls: acc_args.ignore_nulls,
+ order_bys: acc_args.order_bys,
+ is_reversed: acc_args.is_reversed,
+ name: acc_args.name,
+ is_distinct: acc_args.is_distinct,
};
let approx_percentile_cont_accumulator =
self.approx_percentile_cont.create_accumulator(sub_args)?;
diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs
index 4d8676f24a..b830588d40 100644
--- a/datafusion/functions-aggregate/src/array_agg.rs
+++ b/datafusion/functions-aggregate/src/array_agg.rs
@@ -162,9 +162,9 @@ impl AggregateUDFImpl for ArrayAgg {
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
- let data_type = acc_args.exprs[0].data_type(acc_args.schema)?;
- let ignore_nulls =
- acc_args.ignore_nulls && acc_args.exprs[0].nullable(acc_args.schema)?;
+ let field = &acc_args.expr_fields[0];
+ let data_type = field.data_type();
+ let ignore_nulls = acc_args.ignore_nulls && field.is_nullable();
if acc_args.is_distinct {
// Limitation similar to Postgres. The aggregation function can only mix
@@ -191,7 +191,7 @@ impl AggregateUDFImpl for ArrayAgg {
}
};
return Ok(Box::new(DistinctArrayAggAccumulator::try_new(
- &data_type,
+ data_type,
sort_option,
ignore_nulls,
)?));
@@ -199,7 +199,7 @@ impl AggregateUDFImpl for ArrayAgg {
let Some(ordering) = LexOrdering::new(acc_args.order_bys.to_vec()) else {
return Ok(Box::new(ArrayAggAccumulator::try_new(
- &data_type,
+ data_type,
ignore_nulls,
)?));
};
@@ -210,7 +210,7 @@ impl AggregateUDFImpl for ArrayAgg {
.collect::<Result<Vec<_>>>()?;
OrderSensitiveArrayAggAccumulator::try_new(
- &data_type,
+ data_type,
&ordering_dtypes,
ordering,
self.is_input_pre_ordered,
@@ -802,6 +802,7 @@ mod tests {
use datafusion_common::cast::as_generic_string_array;
use datafusion_common::internal_err;
use datafusion_physical_expr::expressions::Column;
+ use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use std::sync::Arc;
@@ -1159,15 +1160,18 @@ mod tests {
}
fn build(&self) -> Result<Box<dyn Accumulator>> {
+ let expr = Arc::new(Column::new("col", 0));
+ let expr_field = expr.return_field(&self.schema)?;
ArrayAgg::default().accumulator(AccumulatorArgs {
return_field: Arc::clone(&self.return_field),
schema: &self.schema,
+ expr_fields: &[expr_field],
ignore_nulls: false,
order_bys: &self.order_bys,
is_reversed: false,
name: "",
is_distinct: self.distinct,
- exprs: &[Arc::new(Column::new("col", 0))],
+ exprs: &[expr],
})
}
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs
index 11960779ed..bec1734e2e 100644
--- a/datafusion/functions-aggregate/src/average.rs
+++ b/datafusion/functions-aggregate/src/average.rs
@@ -184,12 +184,12 @@ impl AggregateUDFImpl for Avg {
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
- let data_type = acc_args.exprs[0].data_type(acc_args.schema)?;
+ let data_type = acc_args.expr_fields[0].data_type();
use DataType::*;
// instantiate specialized accumulator based for the type
if acc_args.is_distinct {
- match (&data_type, acc_args.return_type()) {
+ match (data_type, acc_args.return_type()) {
// Numeric types are converted to Float64 via `coerce_avg_type` during logical plan creation
(Float64, _) => Ok(Box::new(Float64DistinctAvgAccumulator::default())),
@@ -362,12 +362,13 @@ impl AggregateUDFImpl for Avg {
) -> Result<Box<dyn GroupsAccumulator>> {
use DataType::*;
- let data_type = args.exprs[0].data_type(args.schema)?;
+ let data_type = args.expr_fields[0].data_type();
+
// instantiate specialized accumulator based for the type
- match (&data_type, args.return_field.data_type()) {
+ match (data_type, args.return_field.data_type()) {
(Float64, Float64) => {
Ok(Box::new(AvgGroupsAccumulator::<Float64Type, _>::new(
- &data_type,
+ data_type,
args.return_field.data_type(),
|sum: f64, count: u64| Ok(sum / count as f64),
)))
@@ -386,7 +387,7 @@ impl AggregateUDFImpl for Avg {
move |sum: i32, count: u64| decimal_averager.avg(sum, count as i32);
Ok(Box::new(AvgGroupsAccumulator::<Decimal32Type, _>::new(
- &data_type,
+ data_type,
args.return_field.data_type(),
avg_fn,
)))
@@ -405,7 +406,7 @@ impl AggregateUDFImpl for Avg {
move |sum: i64, count: u64| decimal_averager.avg(sum, count as i64);
Ok(Box::new(AvgGroupsAccumulator::<Decimal64Type, _>::new(
- &data_type,
+ data_type,
args.return_field.data_type(),
avg_fn,
)))
@@ -424,7 +425,7 @@ impl AggregateUDFImpl for Avg {
move |sum: i128, count: u64| decimal_averager.avg(sum, count as i128);
Ok(Box::new(AvgGroupsAccumulator::<Decimal128Type, _>::new(
- &data_type,
+ data_type,
args.return_field.data_type(),
avg_fn,
)))
@@ -445,7 +446,7 @@ impl AggregateUDFImpl for Avg {
};
Ok(Box::new(AvgGroupsAccumulator::<Decimal256Type, _>::new(
- &data_type,
+ data_type,
args.return_field.data_type(),
avg_fn,
)))
@@ -459,7 +460,7 @@ impl AggregateUDFImpl for Avg {
DurationSecondType,
_,
>::new(
- &data_type,
+ data_type,
args.return_type(),
avg_fn,
))),
@@ -467,7 +468,7 @@ impl AggregateUDFImpl for Avg {
DurationMillisecondType,
_,
>::new(
- &data_type,
+ data_type,
args.return_type(),
avg_fn,
))),
@@ -475,7 +476,7 @@ impl AggregateUDFImpl for Avg {
DurationMicrosecondType,
_,
>::new(
- &data_type,
+ data_type,
args.return_type(),
avg_fn,
))),
@@ -483,7 +484,7 @@ impl AggregateUDFImpl for Avg {
DurationNanosecondType,
_,
>::new(
- &data_type,
+ data_type,
args.return_type(),
avg_fn,
))),
diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rs
index c0d2ba199a..065635a891 100644
--- a/datafusion/functions-aggregate/src/count.rs
+++ b/datafusion/functions-aggregate/src/count.rs
@@ -333,7 +333,7 @@ impl AggregateUDFImpl for Count {
return not_impl_err!("COUNT DISTINCT with multiple arguments");
}
- let data_type = &acc_args.exprs[0].data_type(acc_args.schema)?;
+ let data_type = acc_args.expr_fields[0].data_type();
Ok(match data_type {
DataType::Dictionary(_, values_type) => {
@@ -854,7 +854,7 @@ mod tests {
datatypes::{DataType, Field, Int32Type, Schema},
};
use datafusion_expr::function::AccumulatorArgs;
- use datafusion_physical_expr::expressions::Column;
+ use datafusion_physical_expr::{expressions::Column, PhysicalExpr};
use std::sync::Arc;
/// Helper function to create a dictionary array with non-null keys but some null values
/// Returns a dictionary array where:
@@ -895,8 +895,10 @@ mod tests {
// Using Count UDAF's accumulator
let count = Count::new();
let expr = Arc::new(Column::new("dict_col", 0));
+ let expr_field = expr.return_field(&schema)?;
let args = AccumulatorArgs {
schema: &schema,
+ expr_fields: &[expr_field],
exprs: &[expr],
is_distinct: true,
name: "count",
diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs
index a65759594e..9466c6affb 100644
--- a/datafusion/functions-aggregate/src/median.rs
+++ b/datafusion/functions-aggregate/src/median.rs
@@ -162,7 +162,7 @@ impl AggregateUDFImpl for Median {
};
}
- let dt = acc_args.exprs[0].data_type(acc_args.schema)?;
+ let dt = acc_args.expr_fields[0].data_type().clone();
downcast_integer! {
dt => (helper, dt),
DataType::Float16 => helper!(Float16Type, dt),
@@ -196,7 +196,7 @@ impl AggregateUDFImpl for Median {
);
}
- let dt = args.exprs[0].data_type(args.schema)?;
+ let dt = args.expr_fields[0].data_type().clone();
macro_rules! helper {
($t:ty, $dt:expr) => {
diff --git a/datafusion/functions-aggregate/src/nth_value.rs b/datafusion/functions-aggregate/src/nth_value.rs
index b9dc498ee7..2f4f9371be 100644
--- a/datafusion/functions-aggregate/src/nth_value.rs
+++ b/datafusion/functions-aggregate/src/nth_value.rs
@@ -160,8 +160,8 @@ impl AggregateUDFImpl for NthValueAgg {
.map(|e| e.expr.data_type(acc_args.schema))
.collect::<Result<Vec<_>>>()?;
- let data_type = acc_args.exprs[0].data_type(acc_args.schema)?;
- NthValueAccumulator::try_new(n, &data_type, &ordering_dtypes, ordering)
+ let data_type = acc_args.expr_fields[0].data_type();
+ NthValueAccumulator::try_new(n, data_type, &ordering_dtypes, ordering)
.map(|acc| Box::new(acc) as _)
}
diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rs
index 312d5f11b4..782524aa4d 100644
--- a/datafusion/functions-aggregate/src/stddev.rs
+++ b/datafusion/functions-aggregate/src/stddev.rs
@@ -443,26 +443,31 @@ mod tests {
agg2: Arc<AggregateUDF>,
schema: &Schema,
) -> Result<ScalarValue> {
+ let expr = col("a", schema)?;
+ let expr_field = expr.return_field(schema)?;
+
let args1 = AccumulatorArgs {
return_field: Field::new("f", DataType::Float64, true).into(),
schema,
+ expr_fields: &[Arc::clone(&expr_field)],
ignore_nulls: false,
order_bys: &[],
name: "a",
is_distinct: false,
is_reversed: false,
- exprs: &[col("a", schema)?],
+ exprs: &[Arc::clone(&expr)],
};
let args2 = AccumulatorArgs {
return_field: Field::new("f", DataType::Float64, true).into(),
schema,
+ expr_fields: &[expr_field],
ignore_nulls: false,
order_bys: &[],
name: "a",
is_distinct: false,
is_reversed: false,
- exprs: &[col("a", schema)?],
+ exprs: &[expr],
};
let mut accum1 = agg1.accumulator(args1)?;
diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs
index a091ed34da..4a040df7b4 100644
--- a/datafusion/functions-aggregate/src/string_agg.rs
+++ b/datafusion/functions-aggregate/src/string_agg.rs
@@ -199,7 +199,16 @@ impl AggregateUDFImpl for StringAgg {
)
.into(),
exprs: &filter_index(acc_args.exprs, 1),
- ..acc_args
+ expr_fields: &filter_index(acc_args.expr_fields, 1),
+ // Unchanged below; we list each field explicitly in case we ever add more
+ // fields to AccumulatorArgs making it easier to see if changes are also
+ // needed here.
+ schema: acc_args.schema,
+ ignore_nulls: acc_args.ignore_nulls,
+ order_bys: acc_args.order_bys,
+ is_reversed: acc_args.is_reversed,
+ name: acc_args.name,
+ is_distinct: acc_args.is_distinct,
})?;
Ok(Box::new(StringAggAccumulator::new(
@@ -590,6 +599,10 @@ mod tests {
StringAgg::new().accumulator(AccumulatorArgs {
return_field: Field::new("f", DataType::LargeUtf8, true).into(),
schema: &self.schema,
+ expr_fields: &[
+ Field::new("col", DataType::LargeUtf8, true).into(),
+ Field::new("lit", DataType::Utf8, false).into(),
+ ],
ignore_nulls: false,
order_bys: &self.order_bys,
is_reversed: false,
diff --git a/datafusion/physical-expr/src/aggregate.rs b/datafusion/physical-expr/src/aggregate.rs
index 19d2ecc924..2a8467eb88 100644
--- a/datafusion/physical-expr/src/aggregate.rs
+++ b/datafusion/physical-expr/src/aggregate.rs
@@ -143,7 +143,7 @@ impl AggregateExprBuilder {
/// # fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
/// # unimplemented!()
/// # }
- /// #
+ /// #
/// # fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
/// # unimplemented!()
/// # }
@@ -231,9 +231,15 @@ impl AggregateExprBuilder {
Some(alias) => alias,
};
+ let arg_fields = args
+ .iter()
+ .map(|e| e.return_field(schema.as_ref()))
+ .collect::<Result<Vec<_>>>()?;
+
Ok(AggregateFunctionExpr {
fun: Arc::unwrap_or_clone(fun),
args,
+ arg_fields,
return_field,
name,
human_display,
@@ -306,6 +312,8 @@ impl AggregateExprBuilder {
pub struct AggregateFunctionExpr {
fun: AggregateUDF,
args: Vec<Arc<dyn PhysicalExpr>>,
+ /// Fields corresponding to args (same order & length)
+ arg_fields: Vec<FieldRef>,
/// Output / return field of this aggregate
return_field: FieldRef,
/// Output column name that this expression creates
@@ -383,6 +391,7 @@ impl AggregateFunctionExpr {
let acc_args = AccumulatorArgs {
return_field: Arc::clone(&self.return_field),
schema: &self.schema,
+ expr_fields: &self.arg_fields,
ignore_nulls: self.ignore_nulls,
order_bys: self.order_bys.as_ref(),
is_distinct: self.is_distinct,
@@ -467,6 +476,7 @@ impl AggregateFunctionExpr {
let args = AccumulatorArgs {
return_field: Arc::clone(&self.return_field),
schema: &self.schema,
+ expr_fields: &self.arg_fields,
ignore_nulls: self.ignore_nulls,
order_bys: self.order_bys.as_ref(),
is_distinct: self.is_distinct,
@@ -536,6 +546,7 @@ impl AggregateFunctionExpr {
let args = AccumulatorArgs {
return_field: Arc::clone(&self.return_field),
schema: &self.schema,
+ expr_fields: &self.arg_fields,
ignore_nulls: self.ignore_nulls,
order_bys: self.order_bys.as_ref(),
is_distinct: self.is_distinct,
@@ -555,6 +566,7 @@ impl AggregateFunctionExpr {
let args = AccumulatorArgs {
return_field: Arc::clone(&self.return_field),
schema: &self.schema,
+ expr_fields: &self.arg_fields,
ignore_nulls: self.ignore_nulls,
order_bys: self.order_bys.as_ref(),
is_distinct: self.is_distinct,
@@ -638,6 +650,9 @@ impl AggregateFunctionExpr {
Some(AggregateFunctionExpr {
fun: self.fun.clone(),
args,
+ // TODO: need to align arg_fields here with new args
+ // https://github.com/apache/datafusion/issues/18149
+ arg_fields: self.arg_fields.clone(),
return_field: Arc::clone(&self.return_field),
name: self.name.clone(),
// TODO: Human name should be updated after re-write to not mislead