Apache DataFusion SQL Query Engine
DataFrame API: allow aggregate functions in select() 2 (#17874) (#25482)
Sergey Zhukov0cba642ceff7
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 0cf86aa969..d0738b9897 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -51,12 +51,15 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::util::display::{ArrayFormatter, FormatOptions};use arrow_schema::FieldRef;use datafusion_common::config::{CsvOptions, JsonOptions};+use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};use datafusion_common::{Column, DFSchema, DataFusionError, ParamValues, ScalarValue, SchemaError,TableReference, UnnestOptions, exec_err, internal_datafusion_err, not_impl_err,plan_datafusion_err, plan_err, unqualified_field_not_found,};+use datafusion_expr::expr_rewriter::{normalize_col, normalize_cols};use datafusion_expr::select_expr::SelectExpr;+use datafusion_expr::utils::{columnize_expr, find_aggregate_exprs};use datafusion_expr::{ExplainOption, ScalarUDF, SortExpr, TableProviderFilterPushDown, UNNAMED_TABLE, case,dml::InsertOp, is_null, lit, utils::COUNT_STAR_EXPANSION,@@ -249,6 +252,113 @@ pub struct DataFrame {projection_requires_validation: bool,}+/// True when select() should insert a global Aggregate before projecting.+///+/// Requires a pure aggregate list (no wildcards or input columns outside+/// aggregate calls). Aggregates that already columnize onto plan (for+/// example after [`DataFrame::aggregate`]) stay on the projection path.+fn should_apply_global_aggregate(+ expr_list: &[SelectExpr],+ plan: &LogicalPlan,+) -> Result<bool> {+ let mut remaining_aggs = false;+ for expr in expr_list {+ match expr {+ SelectExpr::Wildcard(_) | SelectExpr::QualifiedWildcard(_, _) => {+ return Ok(false);+ }+ SelectExpr::Expression(expr) => {+ if has_column_outside_aggregate(expr) {+ return Ok(false);+ }+ let columnized =+ columnize_expr(normalize_col(expr.clone(), plan)?, plan)?;+ if !find_aggregate_exprs([&columnized]).is_empty() {+ remaining_aggs = true;+ }+ }+ }+ }+ // False when every aggregate already columnizes onto plan.+ Ok(remaining_aggs)+}++/// Alias each aggregate uniquely so the Aggregate schema has distinct names.+///+/// The aliases are internal: the projection built on top of the Aggregate+/// re-aliases its outputs, so user-facing names are unaffected. Identical+/// aggregates are already deduplicated by find_aggregate_exprs; the suffix+/// loop only guards against distinct aggregates whose names collide.+///+/// Returns the aliased aggregates and a map from the normalized original+/// expression to the output column.+fn alias_aggregate_exprs(+ exprs: Vec<Expr>,+ plan: &LogicalPlan,+) -> Result<(Vec<Expr>, HashMap<Expr, Expr>)> {+ let exprs = normalize_cols(exprs, plan)?;+ let mut used = HashSet::new();+ let mut rewrite_map = HashMap::new();+ let mut aliased = Vec::with_capacity(exprs.len());+ for expr in exprs {+ if rewrite_map.contains_key(&expr) {+ continue;+ }+ let base = expr.name_for_alias()?;+ let mut name = base.clone();+ let mut i = 1;+ while !used.insert(name.clone()) {+ name = format!("{base}_{i}");+ i += 1;+ }+ rewrite_map.insert(expr.clone(), Expr::Column(Column::from_name(&name)));+ aliased.push(expr.alias(name));+ }+ Ok((aliased, rewrite_map))+}++/// Replace aggregate subexpressions with columns produced by the Aggregate node.+fn rewrite_select_aggs(+ expr_list: Vec<SelectExpr>,+ input: &LogicalPlan,+ rewrite_map: &HashMap<Expr, Expr>,+) -> Result<Vec<SelectExpr>> {+ expr_list+ .into_iter()+ .map(|select_expr| match select_expr {+ SelectExpr::Expression(expr) => {+ let expr = normalize_col(expr, input)?;+ let expr = expr+ .transform(|node| {+ Ok(match rewrite_map.get(&node) {+ Some(col) => Transformed::yes(col.clone()),+ None => Transformed::no(node),+ })+ })?+ .data;+ Ok(SelectExpr::Expression(expr))+ }+ other => Ok(other),+ })+ .collect()+}++/// True if expr references an input column that is not inside an aggregate.+fn has_column_outside_aggregate(expr: &Expr) -> bool {+ let mut found = false;+ let _ = expr.apply(|e| {+ if matches!(e, Expr::AggregateFunction(_)) {+ return Ok(TreeNodeRecursion::Jump);+ }+ if matches!(e, Expr::Column(_)) {+ found = true;+ return Ok(TreeNodeRecursion::Stop);+ }+ Ok(TreeNodeRecursion::Continue)+ });+ found+}+impl DataFrame {/// Create a new `DataFrame ` based on an existing `LogicalPlan`///@@ -380,15 +490,23 @@ impl DataFrame {self.select(expr_list)}- /// Project arbitrary expressions (like SQL SELECT expressions) into a new- /// `DataFrame`.+ /// Project expressions into a new [`DataFrame`], like SQL SELECT.///- /// The output `DataFrame` has one column for each element in `expr_list`.+ /// The output has one column for each element in expr_list.+ ///+ /// A list of only aggregates is treated as a global aggregation+ /// ([`Self::aggregate`] with no group columns). To group, use+ /// [`Self::aggregate`]. Mixing aggregates with non-aggregated columns is+ /// not treated as an implicit GROUP BY.+ ///+ /// Output names must be unique, as with any projection; duplicate names+ /// are rejected rather than renamed.////// # Example/// ```/// # use datafusion::prelude::*;/// # use datafusion::error::Result;+ /// # use datafusion::functions_aggregate::expr_fn::{count, sum};/// # use datafusion_common::assert_batches_sorted_eq;/// # #[tokio::main]/// # async fn main() -> Result<()> {@@ -396,7 +514,9 @@ impl DataFrame {/// let df = ctx/// .read_csv("tests/data/example.csv", CsvReadOptions::new())/// .await?;- /// let df = df.select(vec![col("a"), col("b") * col("c")])?;+ ///+ /// // Per-row projection+ /// let res = df.clone().select(vec![col("a"), col("b") * col("c")])?;/// let expected = vec![/// "+---+-----------------------+",/// "| a | ?table?.b * ?table?.c |",@@ -404,7 +524,21 @@ impl DataFrame {/// "| 1 | 6 |",/// "+---+-----------------------+",/// ];- /// # assert_batches_sorted_eq!(expected, &df.collect().await?);+ /// # assert_batches_sorted_eq!(expected, &res.collect().await?);+ ///+ /// // Global aggregation (no GROUP BY) — same as aggregate(vec![], ...)+ /// let res = df.select(vec![+ /// count(col("a")).alias("cnt"),+ /// sum(col("b")).alias("sum_b"),+ /// ])?;+ /// let expected = vec![+ /// "+-----+-------+",+ /// "| cnt | sum_b |",+ /// "+-----+-------+",+ /// "| 1 | 2 |",+ /// "+-----+-------+",+ /// ];+ /// # assert_batches_sorted_eq!(expected, &res.collect().await?);/// # Ok(())/// # }/// ```@@ -412,21 +546,47 @@ impl DataFrame {self,expr_list: impl IntoIterator<Item = impl Into<SelectExpr>>,) -> Result<DataFrame> {- let expr_list: Vec<SelectExpr> =+ let mut expr_list: Vec<SelectExpr> =expr_list.into_iter().map(|e| e.into()).collect::<Vec<_>>();- let expressions = expr_list.iter().filter_map(|e| match e {- SelectExpr::Expression(expr) => Some(expr),- _ => None,- });+ let expressions: Vec<&Expr> = expr_list+ .iter()+ .filter_map(|e| match e {+ SelectExpr::Expression(expr) => Some(expr),+ _ => None,+ })+ .collect();- let window_func_exprs = find_window_exprs(expressions);- let plan = if window_func_exprs.is_empty() {- self.plan- } else {+ // 1. Windows (row-preserving)+ let window_func_exprs = find_window_exprs(expressions.iter().copied());+ let has_windows = !window_func_exprs.is_empty();+ let mut plan = if has_windows {LogicalPlanBuilder::window_plan(self.plan, window_func_exprs)?+ } else {+ self.plan};+ // 2. Global aggregate only (no GROUP BY). Mixed lists and+ // aggregate().select() reshapes stay on the projection path.+ // Aggregates get unique internal aliases here; the user-facing+ // output names are validated by project() below, like any+ // other projection.+ let aggr_exprs = find_aggregate_exprs(expressions.iter().copied());+ if !aggr_exprs.is_empty()+ && !has_windows+ && should_apply_global_aggregate(&expr_list, &plan)?+ {+ let input = plan;+ let (aggr_with_alias, rewrite_map) =+ alias_aggregate_exprs(aggr_exprs, &input)?;+ plan = LogicalPlanBuilder::from(input.clone())+ .aggregate(Vec::<Expr>::new(), aggr_with_alias)?+ .build()?;+ expr_list = rewrite_select_aggs(expr_list, &input, &rewrite_map)?;+ }++ // 3. Project — columnize_expr maps window/aggregate exprs to columns+ // and duplicate output names are rejectedlet project_plan = LogicalPlanBuilder::from(plan).project(expr_list)?.build()?;Ok(DataFrame {diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 1a4f6ffcb4..4bbf6e2bcc 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -35,6 +35,7 @@ use arrow::util::pretty::pretty_format_batches;use arrow_schema::{SortOptions, TimeUnit};use datafusion::{assert_batches_eq, dataframe};use datafusion_common::metadata::FieldMetadata;+use datafusion_expr::select_expr::SelectExpr;use datafusion_functions_aggregate::count::{count_all, count_all_window};use datafusion_functions_aggregate::expr_fn::{array_agg, avg, avg_distinct, count, count_distinct, max, median, min, sum,@@ -74,7 +75,9 @@ use datafusion_common_runtime::SpawnedTask;use datafusion_datasource::file_format::format_as_file_type;use datafusion_execution::config::SessionConfig;use datafusion_execution::runtime_env::RuntimeEnv;-use datafusion_expr::expr::{GroupingSet, NullTreatment, Sort, WindowFunction};+use datafusion_expr::expr::{+ GroupingSet, NullTreatment, Sort, WildcardOptions, WindowFunction,+};use datafusion_expr::var_provider::{VarProvider, VarType};use datafusion_expr::{CreateMemoryTable, CreateView, DdlStatement, Expr, ExprFunctionExt, ExprSchemable,@@ -7998,3 +8001,492 @@ async fn test_unresolved_lambda_variable() -> Result<()> {Ok(())}++#[tokio::test]+async fn test_dataframe_api_select_semantics() -> Result<()> {+ let df = test_table().await?;++ // ----------------------------------------------------------------------+ // Aggregate functions in SELECT (no GROUP BY)+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT+ // COUNT(c9) AS count_c9,+ // COUNT(CAST(c9 AS Utf8View)) AS count_c9_str,+ // SUM(c9) AS sum_c9,+ // COUNT(c8) AS count_c8,+ // SUM(c9) + COUNT(c8) AS total1,+ // (COUNT(c9) + 1) * 2 AS total2,+ // COUNT(c9) + 1 AS count_c9_add_1+ // FROM t+ let res = df.clone().select(vec![+ count(col("c9")).alias("count_c9"),+ count(cast(col("c9"), DataType::Utf8View)).alias("count_c9_str"),+ sum(col("c9")).alias("sum_c9"),+ count(col("c8")).alias("count_c8"),+ (sum(col("c9")) + count(col("c8"))).alias("total1"),+ ((count(col("c9")) + lit(1)) * lit(2)).alias("total2"),+ (count(col("c9")) + lit(1)).alias("count_c9_add_1"),+ ])?;++ assert_batches_eq!(+ &[+ "+----------+--------------+--------------+----------+--------------+--------+----------------+",+ "| count_c9 | count_c9_str | sum_c9 | count_c8 | total1 | total2 | count_c9_add_1 |",+ "+----------+--------------+--------------+----------+--------------+--------+----------------+",+ "| 100 | 100 | 222089770060 | 100 | 222089770160 | 202 | 101 |",+ "+----------+--------------+--------------+----------+--------------+--------+----------------+",+ ],+ &res.collect().await?+ );++ // ----------------------------------------------------------------------+ // Aggregate + literal (still one row)+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT SUM(c9) AS sum_c9, 1 AS one FROM t+ let res = df+ .clone()+ .select(vec![sum(col("c9")).alias("sum_c9"), lit(1).alias("one")])?;++ assert_batches_eq!(+ &[+ "+--------------+-----+",+ "| sum_c9 | one |",+ "+--------------+-----+",+ "| 222089770060 | 1 |",+ "+--------------+-----+",+ ],+ &res.collect().await?+ );++ // ----------------------------------------------------------------------+ // Empty input (global aggregate still returns exactly one row)+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT COUNT(c9) AS cnt, SUM(c9) AS sum_c9, 1 AS one FROM t WHERE false+ let res = df.clone().filter(lit(false))?.select(vec![+ count(col("c9")).alias("cnt"),+ sum(col("c9")).alias("sum_c9"),+ lit(1).alias("one"),+ ])?;++ assert_batches_eq!(+ &[+ "+-----+--------+-----+",+ "| cnt | sum_c9 | one |",+ "+-----+--------+-----+",+ "| 0 | | 1 |",+ "+-----+--------+-----+",+ ],+ &res.collect().await?+ );++ // ----------------------------------------------------------------------+ // Duplicate output names are rejected, like any other projection+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT COUNT(c9) AS count_c9, COUNT(c9) AS count_c9 FROM t+ let err = df+ .clone()+ .select(vec![+ count(col("c9")).alias("count_c9"),+ count(col("c9")).alias("count_c9"),+ ])+ .unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains("require unique expression names"),+ "duplicate aliases must be rejected at planning time, got: {msg}"+ );++ // SQL:+ // SELECT COUNT(c9), COUNT(c9) FROM t+ let err = df+ .clone()+ .select(vec![count(col("c9")), count(col("c9"))])+ .unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains("require unique expression names"),+ "duplicate unaliased aggregates must be rejected at planning time, got: {msg}"+ );++ // ----------------------------------------------------------------------+ // Mixed column + aggregate (no implicit GROUP BY)+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT c1, SUM(c9) AS sum_c9 FROM t+ // Must not become GROUP BY c1. Planning succeeds; execution still fails.+ let res = df+ .clone()+ .select(vec![col("c1"), sum(col("c9")).alias("sum_c9")])?;+ let err = res.collect().await.unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains("not supported in this position"),+ "mixed list must not run as grouped aggregation, got: {msg}"+ );++ // ----------------------------------------------------------------------+ // Wildcard handling+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT *, 42 FROM t+ let res = df+ .clone()+ .select(vec![+ SelectExpr::Wildcard(WildcardOptions::default()),+ lit(42).into(),+ ])?+ .limit(0, None)?;++ let batches = res.collect().await?;+ assert!(!batches.is_empty());+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 100);+ assert!(batches.iter().all(|b| b.num_columns() == 14));++ // SQL:+ // SELECT aggregate_test_100.*, 42 FROM t+ let res = df.clone().select(vec![+ SelectExpr::QualifiedWildcard(+ "aggregate_test_100".into(),+ WildcardOptions::default(),+ ),+ lit(42).into(),+ ])?;++ let batches = res.collect().await?;+ assert!(!batches.is_empty());+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 100);+ assert!(batches.iter().all(|b| b.num_columns() == 14));++ // ----------------------------------------------------------------------+ // Window functions+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT+ // c1,+ // COUNT(c9) OVER (PARTITION BY c1) AS cnt,+ // SUM(c9) OVER (PARTITION BY c1) AS sum_c9,+ // AVG(c9) OVER (PARTITION BY c1) AS avg_c9+ // FROM t+ // ORDER BY c1+ let count_window_function = Expr::WindowFunction(Box::new(WindowFunction::new(+ WindowFunctionDefinition::AggregateUDF(+ datafusion_functions_aggregate::count::count_udaf(),+ ),+ vec![col("c9")],+ )))+ .partition_by(vec![col("c1")])+ .order_by(vec![])+ .window_frame(WindowFrame::new(None))+ .build()?;++ let sum_window_function = Expr::WindowFunction(Box::new(WindowFunction::new(+ WindowFunctionDefinition::AggregateUDF(+ datafusion_functions_aggregate::sum::sum_udaf(),+ ),+ vec![col("c9")],+ )))+ .partition_by(vec![col("c1")])+ .order_by(vec![])+ .window_frame(WindowFrame::new(None))+ .build()?;++ let avg_window_function = Expr::WindowFunction(Box::new(WindowFunction::new(+ WindowFunctionDefinition::AggregateUDF(+ datafusion_functions_aggregate::average::avg_udaf(),+ ),+ vec![col("c9")],+ )))+ .partition_by(vec![col("c1")])+ .order_by(vec![])+ .window_frame(WindowFrame::new(None))+ .build()?;++ let res = df+ .clone()+ .select(vec![+ col("c1"),+ count_window_function.alias("cnt"),+ sum_window_function.alias("sum_c9"),+ avg_window_function.alias("avg_c9"),+ ])?+ .sort(vec![col("c1").sort(true, true)])?;+ let batches = res.collect().await?;+ assert!(!batches.is_empty());+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 100);+ assert!(batches.iter().all(|b| b.num_columns() == 4));+ let batch = &batches[0];+ assert_batches_eq!(+ &[+ "+----+-----+-------------+--------------------+",+ "| c1 | cnt | sum_c9 | avg_c9 |",+ "+----+-----+-------------+--------------------+",+ "| a | 21 | 42619217323 | 2029486539.1904762 |",+ "| b | 19 | 42365566310 | 2229766647.894737 |",+ "| c | 21 | 46381998762 | 2208666607.714286 |",+ "| d | 18 | 39910269981 | 2217237221.1666665 |",+ "| e | 21 | 50812717684 | 2419653223.047619 |",+ "+----+-----+-------------+--------------------+",+ ],+ &[+ batch.slice(0, 1), // a+ batch.slice(21, 1), // b+ batch.slice(40, 1), // c+ batch.slice(61, 1), // d+ batch.slice(79, 1) // e+ ]+ );++ // ----------------------------------------------------------------------+ // Window with ORDER BY+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT+ // c1,+ // ROW_NUMBER() OVER (PARTITION BY c1 ORDER BY c9) AS rn+ // FROM t+ // ORDER BY c1+ let row_number_window_function = Expr::WindowFunction(Box::new(WindowFunction::new(+ WindowFunctionDefinition::WindowUDF(+ datafusion::functions_window::row_number::row_number_udwf(),+ ),+ vec![],+ )))+ .partition_by(vec![col("c1")])+ .order_by(vec![col("c9").sort(true, true)])+ .window_frame(WindowFrame::new(None))+ .build()?;++ let res = df+ .clone()+ .select(vec![col("c1"), row_number_window_function.alias("rn")])?+ .sort(vec![col("c1").sort(true, true)])?;++ let batches = res.collect().await?;+ assert!(!batches.is_empty());+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 100);+ assert!(batches.iter().all(|b| b.num_columns() == 2));+ assert_batches_eq!(+ &[+ "+----+----+",+ "| c1 | rn |",+ "+----+----+",+ "| a | 1 |",+ "| a | 2 |",+ "| a | 3 |",+ "| a | 4 |",+ "| a | 5 |",+ "| a | 6 |",+ "| a | 7 |",+ "| a | 8 |",+ "| a | 9 |",+ "| a | 10 |",+ "| a | 11 |",+ "| a | 12 |",+ "| a | 13 |",+ "| a | 14 |",+ "| a | 15 |",+ "| a | 16 |",+ "| a | 17 |",+ "| a | 18 |",+ "| a | 19 |",+ "| a | 20 |",+ "| a | 21 |",+ "| b | 1 |",+ "| b | 2 |",+ "| b | 3 |",+ "| b | 4 |",+ "| b | 5 |",+ "| b | 6 |",+ "| b | 7 |",+ "| b | 8 |",+ "| b | 9 |",+ "| b | 10 |",+ "| b | 11 |",+ "| b | 12 |",+ "| b | 13 |",+ "| b | 14 |",+ "| b | 15 |",+ "| b | 16 |",+ "| b | 17 |",+ "| b | 18 |",+ "| b | 19 |",+ "| c | 1 |",+ "| c | 2 |",+ "| c | 3 |",+ "| c | 4 |",+ "| c | 5 |",+ "| c | 6 |",+ "| c | 7 |",+ "| c | 8 |",+ "| c | 9 |",+ "| c | 10 |",+ "| c | 11 |",+ "| c | 12 |",+ "| c | 13 |",+ "| c | 14 |",+ "| c | 15 |",+ "| c | 16 |",+ "| c | 17 |",+ "| c | 18 |",+ "| c | 19 |",+ "| c | 20 |",+ "| c | 21 |",+ "| d | 1 |",+ "| d | 2 |",+ "| d | 3 |",+ "| d | 4 |",+ "| d | 5 |",+ "| d | 6 |",+ "| d | 7 |",+ "| d | 8 |",+ "| d | 9 |",+ "| d | 10 |",+ "| d | 11 |",+ "| d | 12 |",+ "| d | 13 |",+ "| d | 14 |",+ "| d | 15 |",+ "| d | 16 |",+ "| d | 17 |",+ "| d | 18 |",+ "| e | 1 |",+ "| e | 2 |",+ "| e | 3 |",+ "| e | 4 |",+ "| e | 5 |",+ "| e | 6 |",+ "| e | 7 |",+ "| e | 8 |",+ "| e | 9 |",+ "| e | 10 |",+ "| e | 11 |",+ "| e | 12 |",+ "| e | 13 |",+ "| e | 14 |",+ "| e | 15 |",+ "| e | 16 |",+ "| e | 17 |",+ "| e | 18 |",+ "| e | 19 |",+ "| e | 20 |",+ "| e | 21 |",+ "+----+----+",+ ],+ &batches+ );++ // ----------------------------------------------------------------------+ // Window inside expression+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT+ // c1,+ // COUNT(c9) OVER (PARTITION BY c1) + 1 AS cnt_plus+ // FROM t+ // ORDER BY c1+ let count_window_function = Expr::WindowFunction(Box::new(WindowFunction::new(+ WindowFunctionDefinition::AggregateUDF(+ datafusion_functions_aggregate::count::count_udaf(),+ ),+ vec![col("c9")],+ )))+ .partition_by(vec![col("c1")])+ .order_by(vec![])+ .window_frame(WindowFrame::new(None))+ .build()?;++ let cnt_plus_expr = count_window_function + lit(1);++ let res = df+ .clone()+ .select(vec![col("c1"), cnt_plus_expr.alias("cnt_plus")])?+ .sort(vec![col("c1").sort(true, true)])?;+ let batches = res.collect().await?;+ assert!(!batches.is_empty());+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 100);+ assert!(batches.iter().all(|b| b.num_columns() == 2));+ let batch = &batches[0];+ assert_batches_eq!(+ &[+ "+----+----------+",+ "| c1 | cnt_plus |",+ "+----+----------+",+ "| a | 22 |",+ "| b | 20 |",+ "| c | 22 |",+ "| d | 19 |",+ "| e | 22 |",+ "+----+----------+",+ ],+ &[+ batch.slice(0, 1), // a+ batch.slice(21, 1), // b+ batch.slice(40, 1), // c+ batch.slice(61, 1), // d+ batch.slice(79, 1) // e+ ]+ );++ // ----------------------------------------------------------------------+ // Window functions with mixed frames+ // ----------------------------------------------------------------------+ // SQL:+ // SELECT+ // c1,+ // SUM(c9) OVER () AS total_sum,+ // COUNT(c9) OVER (PARTITION BY c1) AS cnt+ // FROM t+ let sum_window_function = Expr::WindowFunction(Box::new(WindowFunction::new(+ WindowFunctionDefinition::AggregateUDF(+ datafusion_functions_aggregate::sum::sum_udaf(),+ ),+ vec![col("c9")],+ )))+ .partition_by(vec![])+ .order_by(vec![])+ .window_frame(WindowFrame::new(None))+ .build()?;++ let count_window_function = Expr::WindowFunction(Box::new(WindowFunction::new(+ WindowFunctionDefinition::AggregateUDF(+ datafusion_functions_aggregate::count::count_udaf(),+ ),+ vec![col("c9")],+ )))+ .partition_by(vec![col("c1")])+ .order_by(vec![])+ .window_frame(WindowFrame::new(None))+ .build()?;++ let res = df+ .clone()+ .select(vec![+ col("c1"),+ sum_window_function.alias("total_sum"),+ count_window_function.alias("cnt"),+ ])?+ .sort(vec![col("c1").sort(true, true)])?;++ let batches = res.collect().await?;+ assert!(!batches.is_empty());+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 100);+ assert!(batches.iter().all(|b| b.num_columns() == 3));+ let batch = &batches[0];++ assert_batches_eq!(+ &[+ "+----+--------------+-----+",+ "| c1 | total_sum | cnt |",+ "+----+--------------+-----+",+ "| a | 222089770060 | 21 |",+ "+----+--------------+-----+"+ ],+ &[batch.slice(0, 1)]+ );++ Ok(())+}