Apache DataFusion SQL Query Engine
docs(dataframe): add dataframe aggregation example (#26597) (#26600)
Sergey Zhukovda11e68a327b
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex c8bc380ecc..0cc6be9b44 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -110,11 +110,12 @@ cargo run --example dataframe -- dataframe#### Category: Single Process-| Subcommand | File Path | Description |-| --------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------- |-| cache_factory | [`dataframe/cache_factory.rs`](examples/dataframe/cache_factory.rs) | Custom lazy caching for DataFrames using `CacheFactory` |-| dataframe | [`dataframe/dataframe.rs`](examples/dataframe/dataframe.rs) | Query DataFrames from various sources and write output |-| deserialize_to_struct | [`dataframe/deserialize_to_struct.rs`](examples/dataframe/deserialize_to_struct.rs) | Convert Arrow arrays into Rust structs |+| Subcommand | File Path | Description |+| --------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------ |+| aggregate | [`dataframe/aggregate.rs`](examples/dataframe/aggregate.rs) | Global and grouped aggregation with `select` and `aggregate` |+| cache_factory | [`dataframe/cache_factory.rs`](examples/dataframe/cache_factory.rs) | Custom lazy caching for DataFrames using `CacheFactory` |+| dataframe | [`dataframe/dataframe.rs`](examples/dataframe/dataframe.rs) | Query DataFrames from various sources and write output |+| deserialize_to_struct | [`dataframe/deserialize_to_struct.rs`](examples/dataframe/deserialize_to_struct.rs) | Convert Arrow arrays into Rust structs |## Execution Monitoring Examplesdiff --git a/datafusion-examples/examples/dataframe/aggregate.rs b/datafusion-examples/examples/dataframe/aggregate.rsnew file mode 100644index 0000000000..248444c056--- /dev/null+++ b/datafusion-examples/examples/dataframe/aggregate.rs@@ -0,0 +1,170 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! See `main.rs` for how to run it.++use datafusion::error::Result;+use datafusion::functions_aggregate::expr_fn::{avg, count, max, min};+use datafusion::functions_aggregate::min_max::max_udaf;+use datafusion::logical_expr::expr::WindowFunction;+use datafusion::prelude::*;+use datafusion_examples::utils::datasets::ExampleDataset;++/// This example demonstrates aggregation with the DataFrame API.+///+/// `DataFrame::select` accepts aggregate expressions: a list made only of+/// aggregates (and scalar expressions over them) is planned as a global+/// aggregation, exactly like `aggregate(vec![], ...)`. Grouping still goes+/// through `aggregate(group_by, aggs)`.+///+/// * [global_aggregate]: `aggregate(vec![], aggs)` vs `select(aggs)`+/// * [expressions_over_aggregates]: compute on top of aggregates in a single `select`+/// * [grouped_aggregate]: GROUP BY uses `aggregate`; `select` never infers a GROUP BY+/// * [aggregate_vs_window]: aggregate (one row) vs window (one row per input row)+pub async fn aggregate_example() -> Result<()> {+ let ctx = SessionContext::new();+ let dataset = ExampleDataset::Cars;+ ctx.register_csv("cars", dataset.path_str()?, CsvReadOptions::default())+ .await?;++ global_aggregate(&ctx).await?;+ expressions_over_aggregates(&ctx).await?;+ grouped_aggregate(&ctx).await?;+ aggregate_vs_window(&ctx).await?;+ Ok(())+}++/// SELECT count(car) AS n,+/// avg(speed) AS avg_speed,+/// max(speed) AS max_speed+/// FROM cars+async fn global_aggregate(ctx: &SessionContext) -> Result<()> {+ let df = ctx.table("cars").await?;+ let aggs = vec![+ count(col("car")).alias("n"),+ avg(col("speed")).alias("avg_speed"),+ max(col("speed")).alias("max_speed"),+ ];++ // The explicit form: `aggregate` with no group expressions+ println!("global aggregation via aggregate(vec![], aggs)");+ df.clone().aggregate(vec![], aggs.clone())?.show().await?;++ // The same query via `select`: a list containing only aggregates is a+ // global aggregation and returns exactly one row+ println!("global aggregation via select(aggs)");+ df.select(aggs)?.show().await?;++ Ok(())+}++/// SELECT max(speed) - min(speed) AS speed_range,+/// avg(speed) * 3.6 AS avg_speed_kmh+/// FROM cars+async fn expressions_over_aggregates(ctx: &SessionContext) -> Result<()> {+ let df = ctx.table("cars").await?;++ // Two steps: `aggregate` computes the aggregates, then `select` builds+ // expressions over their output columns, referenced by alias+ println!("expressions over aggregates via aggregate(...).select(...)");+ df.clone()+ .aggregate(+ vec![],+ vec![+ max(col("speed")).alias("max_speed"),+ min(col("speed")).alias("min_speed"),+ avg(col("speed")).alias("avg_speed"),+ ],+ )?+ .select(vec![+ (col("max_speed") - col("min_speed")).alias("speed_range"),+ (col("avg_speed") * lit(3.6)).alias("avg_speed_kmh"),+ ])?+ .show()+ .await?;++ // One step: `select` accepts scalar expressions over aggregates directly,+ // so no intermediate aliases and no second call are needed+ println!("expressions over aggregates via select(...)");+ df.select(vec![+ (max(col("speed")) - min(col("speed"))).alias("speed_range"),+ (avg(col("speed")) * lit(3.6)).alias("avg_speed_kmh"),+ ])?+ .show()+ .await?;++ Ok(())+}++/// SELECT car,+/// max(speed) AS max_speed+/// FROM cars+/// GROUP BY car+async fn grouped_aggregate(ctx: &SessionContext) -> Result<()> {+ let df = ctx.table("cars").await?;++ // Grouping is always explicit via `aggregate(group_by, aggs)`+ println!("grouped aggregation via aggregate(group_by, aggs)");+ df.aggregate(vec![col("car")], vec![max(col("speed")).alias("max_speed")])?+ .show()+ .await?;++ // `select` never infers a GROUP BY from the non-aggregated columns:+ //+ // df.select(vec![col("car"), max(col("speed"))])+ //+ // is planned as an ordinary projection and fails when executed with+ // "Aggregate function 'max(cars.speed)' is not supported in this position".+ // Use `aggregate(group_by, aggs)` as above for grouped results.++ Ok(())+}++/// SELECT car,+/// speed,+/// max(speed) OVER (PARTITION BY car) AS max_speed+/// FROM cars+async fn aggregate_vs_window(ctx: &SessionContext) -> Result<()> {+ let df = ctx.table("cars").await?;++ // An aggregate in `select` collapses the input to a single row ...+ println!("aggregate in select: one row for the whole table");+ df.clone()+ .select(vec![max(col("speed")).alias("max_speed")])?+ .show()+ .await?;++ // ... whereas the same function as a window expression keeps every input+ // row and attaches the per-partition value to each of them+ let max_over_car = Expr::WindowFunction(Box::new(WindowFunction::new(+ max_udaf(),+ vec![col("speed")],+ )))+ .partition_by(vec![col("car")])+ .build()?;++ println!("window function in select: one row per input row");+ df.select(vec![+ col("car"),+ col("speed"),+ max_over_car.alias("max_speed"),+ ])?+ .show()+ .await?;++ Ok(())+}diff --git a/datafusion-examples/examples/dataframe/main.rs b/datafusion-examples/examples/dataframe/main.rsindex 62503b4899..4afbeccbab 100644--- a/datafusion-examples/examples/dataframe/main.rs+++ b/datafusion-examples/examples/dataframe/main.rs@@ -21,12 +21,15 @@//!//! ## Usage//! ```bash-//! cargo run --example dataframe -- [all|dataframe|deserialize_to_struct|cache_factory]+//! cargo run --example dataframe -- [all|aggregate|dataframe|deserialize_to_struct|cache_factory]//! ```//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module//!+//! - `aggregate`+//! (file: aggregate.rs, desc: Global and grouped aggregation with `select` and `aggregate`)+//!//! - `cache_factory`//! (file: cache_factory.rs, desc: Custom lazy caching for DataFrames using `CacheFactory`)//@@ -36,6 +39,7 @@//! - `deserialize_to_struct`//! (file: deserialize_to_struct.rs, desc: Convert Arrow arrays into Rust structs)+mod aggregate;mod cache_factory;mod dataframe;mod deserialize_to_struct;@@ -48,6 +52,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};#[strum(serialize_all = "snake_case")]enum ExampleKind {All,+ Aggregate,Dataframe,DeserializeToStruct,CacheFactory,@@ -68,6 +73,9 @@ impl ExampleKind {Box::pin(example.run()).await?;}}+ ExampleKind::Aggregate => {+ aggregate::aggregate_example().await?;+ }ExampleKind::Dataframe => {dataframe::dataframe_example().await?;}