Apache DataFusion SQL Query Engine
Consolidate sql operations examples (#18142) (#18743)
Sergey Zhukovb1af8fe9ced2
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 6443d5df4d..937fb779ec 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -90,10 +90,10 @@ cargo run --example dataframe- [`examples/udf/simple_udf.rs`](examples/udf/simple_udf.rs): Define and invoke a User Defined Scalar Function (UDF)- [`examples/udf/simple_udtf.rs`](examples/udf/simple_udtf.rs): Define and invoke a User Defined Table Function (UDTF)- [`examples/udf/simple_udfw.rs`](examples/udf/simple_udwf.rs): Define and invoke a User Defined Window Function (UDWF)-- [`sql_analysis.rs`](examples/sql_analysis.rs): Analyse SQL queries with DataFusion structures-- [`sql_frontend.rs`](examples/sql_frontend.rs): Create LogicalPlans (only) from sql strings-- [`sql_dialect.rs`](examples/sql_dialect.rs): Example of implementing a custom SQL dialect on top of `DFParser`-- [`sql_query.rs`](examples/sql_query.rs): Query data using SQL (in memory `RecordBatches`, local Parquet files)+- [`examples/sql_ops/analysis.rs`](examples/sql_ops/analysis.rs): Analyse SQL queries with DataFusion structures+- [`examples/sql_ops/frontend.rs`](examples/sql_ops/frontend.rs): Create LogicalPlans (only) from sql strings+- [`examples/sql_ops/dialect.rs`](examples/sql_ops/dialect.rs): Example of implementing a custom SQL dialect on top of `DFParser`+- [`examples/sql_ops/query.rs`](examples/sql_ops/query.rs): Query data using SQL (in memory `RecordBatches`, local Parquet files)## Distributeddiff --git a/datafusion-examples/examples/builtin_functions/date_time.rs b/datafusion-examples/examples/builtin_functions/date_time.rsindex 178cba979c..7fb2c25f56 100644--- a/datafusion-examples/examples/builtin_functions/date_time.rs+++ b/datafusion-examples/examples/builtin_functions/date_time.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::sync::Arc;use arrow::array::{Date32Array, Int32Array};diff --git a/datafusion-examples/examples/builtin_functions/function_factory.rs b/datafusion-examples/examples/builtin_functions/function_factory.rsindex 5d41e7a260..164c5b892a 100644--- a/datafusion-examples/examples/builtin_functions/function_factory.rs+++ b/datafusion-examples/examples/builtin_functions/function_factory.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::datatypes::DataType;use datafusion::common::tree_node::{Transformed, TreeNode};use datafusion::common::{exec_datafusion_err, exec_err, internal_err, DataFusionError};diff --git a/datafusion-examples/examples/builtin_functions/regexp.rs b/datafusion-examples/examples/builtin_functions/regexp.rsindex 13c0786930..b8e1543160 100644--- a/datafusion-examples/examples/builtin_functions/regexp.rs+++ b/datafusion-examples/examples/builtin_functions/regexp.rs@@ -16,6 +16,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::common::{assert_batches_eq, assert_contains};use datafusion::error::Result;use datafusion::prelude::*;diff --git a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs b/datafusion-examples/examples/custom_data_source/csv_json_opener.rsindex 4205bbcdf8..6fad8f4d58 100644--- a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs+++ b/datafusion-examples/examples/custom_data_source/csv_json_opener.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::sync::Arc;use arrow::datatypes::{DataType, Field, Schema};diff --git a/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs b/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rsindex aca63c4f35..554382ea95 100644--- a/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs+++ b/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::common::test_util::datafusion_test_data;use datafusion::error::Result;use datafusion::prelude::*;diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rsindex 2213d50fcc..72c05c1a23 100644--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::any::Any;use std::collections::{BTreeMap, HashMap};use std::fmt::{self, Debug, Formatter};diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rsindex 31ec2845c6..6d8fd35860 100644--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::sync::Arc;use arrow::array::{record_batch, RecordBatch};diff --git a/datafusion-examples/examples/custom_data_source/custom_file_format.rs b/datafusion-examples/examples/custom_data_source/custom_file_format.rsindex 510fa53c59..05a34d3a83 100644--- a/datafusion-examples/examples/custom_data_source/custom_file_format.rs+++ b/datafusion-examples/examples/custom_data_source/custom_file_format.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::{any::Any, sync::Arc};use arrow::{diff --git a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs b/datafusion-examples/examples/custom_data_source/file_stream_provider.rsindex 55d2cc8cc0..b1f471b5b3 100644--- a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs+++ b/datafusion-examples/examples/custom_data_source/file_stream_provider.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+/// Demonstrates how to use [`FileStreamProvider`] and [`StreamTable`] to stream data/// from a file-like source (FIFO) into DataFusion for continuous querying.///diff --git a/datafusion-examples/examples/data_io/catalog.rs b/datafusion-examples/examples/data_io/catalog.rsindex 11614d0747..e68e48bdea 100644--- a/datafusion-examples/examples/data_io/catalog.rs+++ b/datafusion-examples/examples/data_io/catalog.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! Simple example of a catalog/schema implementation.use async_trait::async_trait;use datafusion::{diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rsindex d93b2167c1..baa0a2e660 100644--- a/datafusion-examples/examples/data_io/json_shredding.rs+++ b/datafusion-examples/examples/data_io/json_shredding.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::any::Any;use std::sync::Arc;diff --git a/datafusion-examples/examples/data_io/parquet_advanced_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rsindex af1e03fe4d..304e490bd6 100644--- a/datafusion-examples/examples/data_io/parquet_advanced_index.rs+++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::any::Any;use std::collections::{HashMap, HashSet};use std::fs::File;diff --git a/datafusion-examples/examples/data_io/parquet_embedded_index.rs b/datafusion-examples/examples/data_io/parquet_embedded_index.rsindex 0275a3141f..88a052546b 100644--- a/datafusion-examples/examples/data_io/parquet_embedded_index.rs+++ b/datafusion-examples/examples/data_io/parquet_embedded_index.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! Embedding and using a custom index in Parquet files//!//! # Backgrounddiff --git a/datafusion-examples/examples/data_io/parquet_encrypted.rs b/datafusion-examples/examples/data_io/parquet_encrypted.rsindex 3f26c436a9..e3070cddde 100644--- a/datafusion-examples/examples/data_io/parquet_encrypted.rs+++ b/datafusion-examples/examples/data_io/parquet_encrypted.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::common::DataFusionError;use datafusion::config::{ConfigFileEncryptionProperties, TableParquetOptions};use datafusion::dataframe::{DataFrame, DataFrameWriteOptions};diff --git a/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs b/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rsindex 3dca002db4..1a9bf56c09 100644--- a/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs+++ b/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};use arrow_schema::SchemaRef;use async_trait::async_trait;diff --git a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs b/datafusion-examples/examples/data_io/parquet_exec_visitor.rsindex e0611fa144..925c202eac 100644--- a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs+++ b/datafusion-examples/examples/data_io/parquet_exec_visitor.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::sync::Arc;use datafusion::datasource::file_format::parquet::ParquetFormat;diff --git a/datafusion-examples/examples/data_io/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rsindex 4cca73b1f9..7c708046f8 100644--- a/datafusion-examples/examples/data_io/parquet_index.rs+++ b/datafusion-examples/examples/data_io/parquet_index.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, Int32Array, RecordBatch, StringArray,UInt64Array,diff --git a/datafusion-examples/examples/data_io/query_http_csv.rs b/datafusion-examples/examples/data_io/query_http_csv.rsindex be27f8ad21..71421e6270 100644--- a/datafusion-examples/examples/data_io/query_http_csv.rs+++ b/datafusion-examples/examples/data_io/query_http_csv.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::error::Result;use datafusion::prelude::*;use object_store::http::HttpBuilder;diff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rsindex e4898e19d4..3e1568fb41 100644--- a/datafusion-examples/examples/data_io/remote_catalog.rs+++ b/datafusion-examples/examples/data_io/remote_catalog.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!/// This example shows how to implement the DataFusion [`CatalogProvider`] API/// for catalogs that are remote (require network access) and/or offer only/// asynchronous APIs such as [Polaris], [Unity], and [Hive].diff --git a/datafusion-examples/examples/flight/client.rs b/datafusion-examples/examples/flight/client.rsindex 031beea47d..484576975a 100644--- a/datafusion-examples/examples/flight/client.rs+++ b/datafusion-examples/examples/flight/client.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::collections::HashMap;use std::sync::Arc;use tonic::transport::Endpoint;diff --git a/datafusion-examples/examples/flight/server.rs b/datafusion-examples/examples/flight/server.rsindex dc75287cf2..e4c05a13e8 100644--- a/datafusion-examples/examples/flight/server.rs+++ b/datafusion-examples/examples/flight/server.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::ipc::writer::{CompressionContext, DictionaryTracker, IpcDataGenerator};use std::sync::Arc;diff --git a/datafusion-examples/examples/flight/sql_server.rs b/datafusion-examples/examples/flight/sql_server.rsindex d86860f9d4..529296fac8 100644--- a/datafusion-examples/examples/flight/sql_server.rs+++ b/datafusion-examples/examples/flight/sql_server.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::array::{ArrayRef, StringArray};use arrow::datatypes::{DataType, Field, Schema};use arrow::ipc::writer::IpcWriteOptions;diff --git a/datafusion-examples/examples/query_planning/analyzer_rule.rs b/datafusion-examples/examples/query_planning/analyzer_rule.rsindex b6c97679cb..df1ea4c1d7 100644--- a/datafusion-examples/examples/query_planning/analyzer_rule.rs+++ b/datafusion-examples/examples/query_planning/analyzer_rule.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};use datafusion::common::config::ConfigOptions;use datafusion::common::tree_node::{Transformed, TreeNode};diff --git a/datafusion-examples/examples/query_planning/expr_api.rs b/datafusion-examples/examples/query_planning/expr_api.rsindex 236ac4319b..225be0ee13 100644--- a/datafusion-examples/examples/query_planning/expr_api.rs+++ b/datafusion-examples/examples/query_planning/expr_api.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::collections::HashMap;use std::sync::Arc;diff --git a/datafusion-examples/examples/query_planning/optimizer_rule.rs b/datafusion-examples/examples/query_planning/optimizer_rule.rsindex 4af5ef50b3..34aef85c64 100644--- a/datafusion-examples/examples/query_planning/optimizer_rule.rs+++ b/datafusion-examples/examples/query_planning/optimizer_rule.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};use arrow::datatypes::DataType;use datafusion::common::tree_node::{Transformed, TreeNode};diff --git a/datafusion-examples/examples/query_planning/parse_sql_expr.rs b/datafusion-examples/examples/query_planning/parse_sql_expr.rsindex 44e6b3cf5f..376120de9d 100644--- a/datafusion-examples/examples/query_planning/parse_sql_expr.rs+++ b/datafusion-examples/examples/query_planning/parse_sql_expr.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::datatypes::{DataType, Field, Schema};use datafusion::common::DFSchema;use datafusion::logical_expr::{col, lit};diff --git a/datafusion-examples/examples/query_planning/plan_to_sql.rs b/datafusion-examples/examples/query_planning/plan_to_sql.rsindex f23f083acd..95e04907d9 100644--- a/datafusion-examples/examples/query_planning/plan_to_sql.rs+++ b/datafusion-examples/examples/query_planning/plan_to_sql.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::common::DFSchemaRef;use datafusion::error::Result;use datafusion::logical_expr::sqlparser::ast::Statement;diff --git a/datafusion-examples/examples/query_planning/planner_api.rs b/datafusion-examples/examples/query_planning/planner_api.rsindex dd3643471e..9b8aa1c2fe 100644--- a/datafusion-examples/examples/query_planning/planner_api.rs+++ b/datafusion-examples/examples/query_planning/planner_api.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::error::Result;use datafusion::logical_expr::LogicalPlan;use datafusion::physical_plan::displayable;diff --git a/datafusion-examples/examples/query_planning/pruning.rs b/datafusion-examples/examples/query_planning/pruning.rsindex 7c42d151c0..d7562139c2 100644--- a/datafusion-examples/examples/query_planning/pruning.rs+++ b/datafusion-examples/examples/query_planning/pruning.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::collections::HashSet;use std::sync::Arc;diff --git a/datafusion-examples/examples/query_planning/thread_pools.rs b/datafusion-examples/examples/query_planning/thread_pools.rsindex 768638d11a..6fc7d51e91 100644--- a/datafusion-examples/examples/query_planning/thread_pools.rs+++ b/datafusion-examples/examples/query_planning/thread_pools.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! This example shows how to use separate thread pools (tokio [`Runtime`]))s to//! run the IO and CPU intensive parts of DataFusion plans.//!diff --git a/datafusion-examples/examples/sql_analysis.rs b/datafusion-examples/examples/sql_ops/analysis.rssimilarity index 98%rename from datafusion-examples/examples/sql_analysis.rsrename to datafusion-examples/examples/sql_ops/analysis.rsindex 4ff669faf1..266c650772 100644--- a/datafusion-examples/examples/sql_analysis.rs+++ b/datafusion-examples/examples/sql_ops/analysis.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! This example shows how to use the structures that DataFusion provides to perform//! Analysis on SQL queries and their plans.//!@@ -32,141 +34,9 @@ use datafusion::{};use test_utils::tpcds::tpcds_schemas;-/// Counts the total number of joins in a plan-fn total_join_count(plan: &LogicalPlan) -> usize {- let mut total = 0;-- // We can use the TreeNode API to walk over a LogicalPlan.- plan.apply(|node| {- // if we encounter a join we update the running count- if matches!(node, LogicalPlan::Join(_)) {- total += 1;- }- Ok(TreeNodeRecursion::Continue)- })- .unwrap();-- total-}--/// Counts the total number of joins in a plan and collects every join tree in-/// the plan with their respective join count.-///-/// Join Tree Definition: the largest subtree consisting entirely of joins-///-/// For example, this plan:-///-/// ```text-/// JOIN-/// / \-/// A JOIN-/// / \-/// B C-/// ```-///-/// has a single join tree `(A-B-C)` which will result in `(2, [2])`-///-/// This plan:-///-/// ```text-/// JOIN-/// / \-/// A GROUP-/// |-/// JOIN-/// / \-/// B C-/// ```-///-/// Has two join trees `(A-, B-C)` which will result in `(2, [1, 1])`-fn count_trees(plan: &LogicalPlan) -> (usize, Vec<usize>) {- // this works the same way as `total_count`, but now when we encounter a Join- // we try to collect it's entire tree- let mut to_visit = vec![plan];- let mut total = 0;- let mut groups = vec![];-- while let Some(node) = to_visit.pop() {- // if we encounter a join, we know were at the root of the tree- // count this tree and recurse on it's inputs- if matches!(node, LogicalPlan::Join(_)) {- let (group_count, inputs) = count_tree(node);- total += group_count;- groups.push(group_count);- to_visit.extend(inputs);- } else {- to_visit.extend(node.inputs());- }- }-- (total, groups)-}--/// Count the entire join tree and return its inputs using TreeNode API-///-/// For example, if this function receives following plan:-///-/// ```text-/// JOIN-/// / \-/// A GROUP-/// |-/// JOIN-/// / \-/// B C-/// ```-///-/// It will return `(1, [A, GROUP])`-fn count_tree(join: &LogicalPlan) -> (usize, Vec<&LogicalPlan>) {- let mut inputs = Vec::new();- let mut total = 0;-- join.apply(|node| {- // Some extra knowledge:- //- // optimized plans have their projections pushed down as far as- // possible, which sometimes results in a projection going in between 2- // subsequent joins giving the illusion these joins are not "related",- // when in fact they are.- //- // This plan:- // JOIN- // / \- // A PROJECTION- // |- // JOIN- // / \- // B C- //- // is the same as:- //- // JOIN- // / \- // A JOIN- // / \- // B C- // we can continue the recursion in this case- if let LogicalPlan::Projection(_) = node {- return Ok(TreeNodeRecursion::Continue);- }-- // any join we count- if matches!(node, LogicalPlan::Join(_)) {- total += 1;- Ok(TreeNodeRecursion::Continue)- } else {- inputs.push(node);- // skip children of input node- Ok(TreeNodeRecursion::Jump)- }- })- .unwrap();-- (total, inputs)-}--#[tokio::main]-async fn main() -> Result<()> {+/// Demonstrates how to analyze a SQL query by counting JOINs and identifying+/// join-trees using DataFusion’s `LogicalPlan` and `TreeNode` API.+pub async fn analysis() -> Result<()> {// To show how we can count the joins in a sql query we'll be using query 88// from the TPC-DS benchmark.//@@ -310,3 +180,136 @@ fromOk(())}++/// Counts the total number of joins in a plan+fn total_join_count(plan: &LogicalPlan) -> usize {+ let mut total = 0;++ // We can use the TreeNode API to walk over a LogicalPlan.+ plan.apply(|node| {+ // if we encounter a join we update the running count+ if matches!(node, LogicalPlan::Join(_)) {+ total += 1;+ }+ Ok(TreeNodeRecursion::Continue)+ })+ .unwrap();++ total+}++/// Counts the total number of joins in a plan and collects every join tree in+/// the plan with their respective join count.+///+/// Join Tree Definition: the largest subtree consisting entirely of joins+///+/// For example, this plan:+///+/// ```text+/// JOIN+/// / \+/// A JOIN+/// / \+/// B C+/// ```+///+/// has a single join tree `(A-B-C)` which will result in `(2, [2])`+///+/// This plan:+///+/// ```text+/// JOIN+/// / \+/// A GROUP+/// |+/// JOIN+/// / \+/// B C+/// ```+///+/// Has two join trees `(A-, B-C)` which will result in `(2, [1, 1])`+fn count_trees(plan: &LogicalPlan) -> (usize, Vec<usize>) {+ // this works the same way as `total_count`, but now when we encounter a Join+ // we try to collect it's entire tree+ let mut to_visit = vec![plan];+ let mut total = 0;+ let mut groups = vec![];++ while let Some(node) = to_visit.pop() {+ // if we encounter a join, we know were at the root of the tree+ // count this tree and recurse on it's inputs+ if matches!(node, LogicalPlan::Join(_)) {+ let (group_count, inputs) = count_tree(node);+ total += group_count;+ groups.push(group_count);+ to_visit.extend(inputs);+ } else {+ to_visit.extend(node.inputs());+ }+ }++ (total, groups)+}++/// Count the entire join tree and return its inputs using TreeNode API+///+/// For example, if this function receives following plan:+///+/// ```text+/// JOIN+/// / \+/// A GROUP+/// |+/// JOIN+/// / \+/// B C+/// ```+///+/// It will return `(1, [A, GROUP])`+fn count_tree(join: &LogicalPlan) -> (usize, Vec<&LogicalPlan>) {+ let mut inputs = Vec::new();+ let mut total = 0;++ join.apply(|node| {+ // Some extra knowledge:+ //+ // optimized plans have their projections pushed down as far as+ // possible, which sometimes results in a projection going in between 2+ // subsequent joins giving the illusion these joins are not "related",+ // when in fact they are.+ //+ // This plan:+ // JOIN+ // / \+ // A PROJECTION+ // |+ // JOIN+ // / \+ // B C+ //+ // is the same as:+ //+ // JOIN+ // / \+ // A JOIN+ // / \+ // B C+ // we can continue the recursion in this case+ if let LogicalPlan::Projection(_) = node {+ return Ok(TreeNodeRecursion::Continue);+ }++ // any join we count+ if matches!(node, LogicalPlan::Join(_)) {+ total += 1;+ Ok(TreeNodeRecursion::Continue)+ } else {+ inputs.push(node);+ // skip children of input node+ Ok(TreeNodeRecursion::Jump)+ }+ })+ .unwrap();++ (total, inputs)+}diff --git a/datafusion-examples/examples/sql_dialect.rs b/datafusion-examples/examples/sql_ops/dialect.rssimilarity index 98%rename from datafusion-examples/examples/sql_dialect.rsrename to datafusion-examples/examples/sql_ops/dialect.rsindex 20b515506f..62a9f3fa56 100644--- a/datafusion-examples/examples/sql_dialect.rs+++ b/datafusion-examples/examples/sql_ops/dialect.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::fmt::Display;use datafusion::error::{DataFusionError, Result};@@ -26,8 +28,7 @@ use datafusion::sql::{/// This example demonstrates how to use the DFParser to parse a statement in a custom way////// This technique can be used to implement a custom SQL dialect, for example.-#[tokio::main]-async fn main() -> Result<()> {+pub async fn dialect() -> Result<()> {let mut my_parser =MyParser::new("COPY source_table TO 'file.fasta' STORED AS FASTA")?;diff --git a/datafusion-examples/examples/sql_frontend.rs b/datafusion-examples/examples/sql_ops/frontend.rssimilarity index 99%rename from datafusion-examples/examples/sql_frontend.rsrename to datafusion-examples/examples/sql_ops/frontend.rsindex 1fc9ce24ec..2cfb9de05c 100644--- a/datafusion-examples/examples/sql_frontend.rs+++ b/datafusion-examples/examples/sql_ops/frontend.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::common::{plan_err, TableReference};use datafusion::config::ConfigOptions;@@ -44,7 +46,7 @@ use std::sync::Arc;////// In this example, we demonstrate how to use the lower level APIs directly,/// which only requires the `datafusion-sql` dependency.-pub fn main() -> Result<()> {+pub fn frontend() -> Result<()> {// First, we parse the SQL string. Note that we use the DataFusion// Parser, which wraps the `sqlparser-rs` SQL parser and adds DataFusion// specific syntax such as `CREATE EXTERNAL TABLE`diff --git a/datafusion-examples/examples/sql_ops/main.rs b/datafusion-examples/examples/sql_ops/main.rsnew file mode 100644index 0000000000..d67b16bf5c--- /dev/null+++ b/datafusion-examples/examples/sql_ops/main.rs@@ -0,0 +1,105 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! # SQL Examples+//!+//! These examples demonstrate SQL operations in DataFusion.+//!+//! ## Usage+//! ```bash+//! cargo run --example sql_ops -- [analysis|dialect|frontend|query]+//! ```+//!+//! Each subcommand runs a corresponding example:+//! - `analysis` — analyse SQL queries with DataFusion structures+//! - `dialect` — implementing a custom SQL dialect on top of DFParser+//! - `frontend` — create LogicalPlans (only) from sql strings+//! - `query` — query data using SQL (in memory RecordBatches, local Parquet files)++mod analysis;+mod dialect;+mod frontend;+mod query;++use std::str::FromStr;++use datafusion::error::{DataFusionError, Result};++enum ExampleKind {+ Analysis,+ Dialect,+ Frontend,+ Query,+}++impl AsRef<str> for ExampleKind {+ fn as_ref(&self) -> &str {+ match self {+ Self::Analysis => "analysis",+ Self::Dialect => "dialect",+ Self::Frontend => "frontend",+ Self::Query => "query",+ }+ }+}++impl FromStr for ExampleKind {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ "analysis" => Ok(Self::Analysis),+ "dialect" => Ok(Self::Dialect),+ "frontend" => Ok(Self::Frontend),+ "query" => Ok(Self::Query),+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),+ }+ }+}++impl ExampleKind {+ const ALL: [Self; 4] = [Self::Analysis, Self::Dialect, Self::Frontend, Self::Query];++ const EXAMPLE_NAME: &str = "sql_ops";++ fn variants() -> Vec<&'static str> {+ Self::ALL.iter().map(|x| x.as_ref()).collect()+ }+}++#[tokio::main]+async fn main() -> Result<()> {+ let usage = format!(+ "Usage: cargo run --example {} -- [{}]",+ ExampleKind::EXAMPLE_NAME,+ ExampleKind::variants().join("|")+ );++ let arg = std::env::args().nth(1).ok_or_else(|| {+ eprintln!("{usage}");+ DataFusionError::Execution("Missing argument".to_string())+ })?;++ match arg.parse::<ExampleKind>()? {+ ExampleKind::Analysis => analysis::analysis().await?,+ ExampleKind::Dialect => dialect::dialect().await?,+ ExampleKind::Frontend => frontend::frontend()?,+ ExampleKind::Query => query::query().await?,+ }++ Ok(())+}diff --git a/datafusion-examples/examples/sql_query.rs b/datafusion-examples/examples/sql_ops/query.rssimilarity index 99%rename from datafusion-examples/examples/sql_query.rsrename to datafusion-examples/examples/sql_ops/query.rsindex 4da07d33d0..3e052f0823 100644--- a/datafusion-examples/examples/sql_query.rs+++ b/datafusion-examples/examples/sql_ops/query.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::arrow::array::{UInt64Array, UInt8Array};use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::arrow::record_batch::RecordBatch;@@ -32,8 +34,7 @@ use std::sync::Arc;////// [`query_memtable`]: a simple query against a [`MemTable`]/// [`query_parquet`]: a simple query against a directory with multiple Parquet files-#[tokio::main]-async fn main() -> Result<()> {+pub async fn query() -> Result<()> {query_memtable().await?;query_parquet().await?;Ok(())diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rsindex 81e227bfac..b4916e5bd0 100644--- a/datafusion-examples/examples/udf/advanced_udaf.rs+++ b/datafusion-examples/examples/udf/advanced_udaf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::datatypes::{Field, Schema};use datafusion::physical_expr::NullState;use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility};diff --git a/datafusion-examples/examples/udf/advanced_udf.rs b/datafusion-examples/examples/udf/advanced_udf.rsindex bb5a68e90c..7459501a1c 100644--- a/datafusion-examples/examples/udf/advanced_udf.rs+++ b/datafusion-examples/examples/udf/advanced_udf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::any::Any;use std::sync::Arc;diff --git a/datafusion-examples/examples/udf/advanced_udwf.rs b/datafusion-examples/examples/udf/advanced_udwf.rsindex 86f215e019..37b6671639 100644--- a/datafusion-examples/examples/udf/advanced_udwf.rs+++ b/datafusion-examples/examples/udf/advanced_udwf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::datatypes::Field;use arrow::{array::{ArrayRef, AsArray, Float64Array},diff --git a/datafusion-examples/examples/udf/async_udf.rs b/datafusion-examples/examples/udf/async_udf.rsindex 475775a599..c55650223c 100644--- a/datafusion-examples/examples/udf/async_udf.rs+++ b/datafusion-examples/examples/udf/async_udf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! This example shows how to create and use "Async UDFs" in DataFusion.//!//! Async UDFs allow you to perform asynchronous operations, such asdiff --git a/datafusion-examples/examples/udf/simple_udaf.rs b/datafusion-examples/examples/udf/simple_udaf.rsindex e9f905e720..42ea0054b7 100644--- a/datafusion-examples/examples/udf/simple_udaf.rs+++ b/datafusion-examples/examples/udf/simple_udaf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!/// In this example we will declare a single-type, single return type UDAF that computes the geometric mean./// The geometric mean is described here: https://en.wikipedia.org/wiki/Geometric_meanuse datafusion::arrow::{diff --git a/datafusion-examples/examples/udf/simple_udf.rs b/datafusion-examples/examples/udf/simple_udf.rsindex 7d4f3588e3..e8d6c9c817 100644--- a/datafusion-examples/examples/udf/simple_udf.rs+++ b/datafusion-examples/examples/udf/simple_udf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use datafusion::{arrow::{array::{ArrayRef, Float32Array, Float64Array},diff --git a/datafusion-examples/examples/udf/simple_udtf.rs b/datafusion-examples/examples/udf/simple_udtf.rsindex a03b157134..12ee74fc52 100644--- a/datafusion-examples/examples/udf/simple_udtf.rs+++ b/datafusion-examples/examples/udf/simple_udtf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::csv::reader::Format;use arrow::csv::ReaderBuilder;use async_trait::async_trait;diff --git a/datafusion-examples/examples/udf/simple_udwf.rs b/datafusion-examples/examples/udf/simple_udwf.rsindex 2cf1df8d8e..a4f6d59a7d 100644--- a/datafusion-examples/examples/udf/simple_udwf.rs+++ b/datafusion-examples/examples/udf/simple_udwf.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::sync::Arc;use arrow::{