Skip to content

Apache DataFusion SQL Query Engine

Consolidate sql operations examples (#18142) (#18743)

Sergey Zhukovb1af8fe9ced2
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
index 6443d5df4d..937fb779ec 100644
--- a/datafusion-examples/README.md
+++ b/datafusion-examples/README.md
@@ -90,10 +90,10 @@ cargo run --example dataframe
- [`examples/udf/simple_udf.rs`](examples/udf/simple_udf.rs): Define and invoke a User Defined Scalar Function (UDF)
- [`examples/udf/simple_udtf.rs`](examples/udf/simple_udtf.rs): Define and invoke a User Defined Table Function (UDTF)
- [`examples/udf/simple_udfw.rs`](examples/udf/simple_udwf.rs): Define and invoke a User Defined Window Function (UDWF)
-- [`sql_analysis.rs`](examples/sql_analysis.rs): Analyse SQL queries with DataFusion structures
-- [`sql_frontend.rs`](examples/sql_frontend.rs): Create LogicalPlans (only) from sql strings
-- [`sql_dialect.rs`](examples/sql_dialect.rs): Example of implementing a custom SQL dialect on top of `DFParser`
-- [`sql_query.rs`](examples/sql_query.rs): Query data using SQL (in memory `RecordBatches`, local Parquet files)
+- [`examples/sql_ops/analysis.rs`](examples/sql_ops/analysis.rs): Analyse SQL queries with DataFusion structures
+- [`examples/sql_ops/frontend.rs`](examples/sql_ops/frontend.rs): Create LogicalPlans (only) from sql strings
+- [`examples/sql_ops/dialect.rs`](examples/sql_ops/dialect.rs): Example of implementing a custom SQL dialect on top of `DFParser`
+- [`examples/sql_ops/query.rs`](examples/sql_ops/query.rs): Query data using SQL (in memory `RecordBatches`, local Parquet files)
## Distributed
diff --git a/datafusion-examples/examples/builtin_functions/date_time.rs b/datafusion-examples/examples/builtin_functions/date_time.rs
index 178cba979c..7fb2c25f56 100644
--- a/datafusion-examples/examples/builtin_functions/date_time.rs
+++ b/datafusion-examples/examples/builtin_functions/date_time.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::sync::Arc;
use arrow::array::{Date32Array, Int32Array};
diff --git a/datafusion-examples/examples/builtin_functions/function_factory.rs b/datafusion-examples/examples/builtin_functions/function_factory.rs
index 5d41e7a260..164c5b892a 100644
--- a/datafusion-examples/examples/builtin_functions/function_factory.rs
+++ b/datafusion-examples/examples/builtin_functions/function_factory.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::datatypes::DataType;
use datafusion::common::tree_node::{Transformed, TreeNode};
use datafusion::common::{exec_datafusion_err, exec_err, internal_err, DataFusionError};
diff --git a/datafusion-examples/examples/builtin_functions/regexp.rs b/datafusion-examples/examples/builtin_functions/regexp.rs
index 13c0786930..b8e1543160 100644
--- a/datafusion-examples/examples/builtin_functions/regexp.rs
+++ b/datafusion-examples/examples/builtin_functions/regexp.rs
@@ -16,6 +16,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::common::{assert_batches_eq, assert_contains};
use datafusion::error::Result;
use datafusion::prelude::*;
diff --git a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs b/datafusion-examples/examples/custom_data_source/csv_json_opener.rs
index 4205bbcdf8..6fad8f4d58 100644
--- a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs
+++ b/datafusion-examples/examples/custom_data_source/csv_json_opener.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::sync::Arc;
use arrow::datatypes::{DataType, Field, Schema};
diff --git a/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs b/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs
index aca63c4f35..554382ea95 100644
--- a/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs
+++ b/datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::common::test_util::datafusion_test_data;
use datafusion::error::Result;
use datafusion::prelude::*;
diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
index 2213d50fcc..72c05c1a23 100644
--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::any::Any;
use std::collections::{BTreeMap, HashMap};
use std::fmt::{self, Debug, Formatter};
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
index 31ec2845c6..6d8fd35860 100644
--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::sync::Arc;
use arrow::array::{record_batch, RecordBatch};
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_format.rs b/datafusion-examples/examples/custom_data_source/custom_file_format.rs
index 510fa53c59..05a34d3a83 100644
--- a/datafusion-examples/examples/custom_data_source/custom_file_format.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_file_format.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::{any::Any, sync::Arc};
use arrow::{
diff --git a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs b/datafusion-examples/examples/custom_data_source/file_stream_provider.rs
index 55d2cc8cc0..b1f471b5b3 100644
--- a/datafusion-examples/examples/custom_data_source/file_stream_provider.rs
+++ b/datafusion-examples/examples/custom_data_source/file_stream_provider.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
/// Demonstrates how to use [`FileStreamProvider`] and [`StreamTable`] to stream data
/// from a file-like source (FIFO) into DataFusion for continuous querying.
///
diff --git a/datafusion-examples/examples/data_io/catalog.rs b/datafusion-examples/examples/data_io/catalog.rs
index 11614d0747..e68e48bdea 100644
--- a/datafusion-examples/examples/data_io/catalog.rs
+++ b/datafusion-examples/examples/data_io/catalog.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! Simple example of a catalog/schema implementation.
use async_trait::async_trait;
use datafusion::{
diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rs
index d93b2167c1..baa0a2e660 100644
--- a/datafusion-examples/examples/data_io/json_shredding.rs
+++ b/datafusion-examples/examples/data_io/json_shredding.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::any::Any;
use std::sync::Arc;
diff --git a/datafusion-examples/examples/data_io/parquet_advanced_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rs
index af1e03fe4d..304e490bd6 100644
--- a/datafusion-examples/examples/data_io/parquet_advanced_index.rs
+++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::any::Any;
use std::collections::{HashMap, HashSet};
use std::fs::File;
diff --git a/datafusion-examples/examples/data_io/parquet_embedded_index.rs b/datafusion-examples/examples/data_io/parquet_embedded_index.rs
index 0275a3141f..88a052546b 100644
--- a/datafusion-examples/examples/data_io/parquet_embedded_index.rs
+++ b/datafusion-examples/examples/data_io/parquet_embedded_index.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! Embedding and using a custom index in Parquet files
//!
//! # Background
diff --git a/datafusion-examples/examples/data_io/parquet_encrypted.rs b/datafusion-examples/examples/data_io/parquet_encrypted.rs
index 3f26c436a9..e3070cddde 100644
--- a/datafusion-examples/examples/data_io/parquet_encrypted.rs
+++ b/datafusion-examples/examples/data_io/parquet_encrypted.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::common::DataFusionError;
use datafusion::config::{ConfigFileEncryptionProperties, TableParquetOptions};
use datafusion::dataframe::{DataFrame, DataFrameWriteOptions};
diff --git a/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs b/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs
index 3dca002db4..1a9bf56c09 100644
--- a/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs
+++ b/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
use arrow_schema::SchemaRef;
use async_trait::async_trait;
diff --git a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs b/datafusion-examples/examples/data_io/parquet_exec_visitor.rs
index e0611fa144..925c202eac 100644
--- a/datafusion-examples/examples/data_io/parquet_exec_visitor.rs
+++ b/datafusion-examples/examples/data_io/parquet_exec_visitor.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::sync::Arc;
use datafusion::datasource::file_format::parquet::ParquetFormat;
diff --git a/datafusion-examples/examples/data_io/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rs
index 4cca73b1f9..7c708046f8 100644
--- a/datafusion-examples/examples/data_io/parquet_index.rs
+++ b/datafusion-examples/examples/data_io/parquet_index.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::array::{
Array, ArrayRef, AsArray, BooleanArray, Int32Array, RecordBatch, StringArray,
UInt64Array,
diff --git a/datafusion-examples/examples/data_io/query_http_csv.rs b/datafusion-examples/examples/data_io/query_http_csv.rs
index be27f8ad21..71421e6270 100644
--- a/datafusion-examples/examples/data_io/query_http_csv.rs
+++ b/datafusion-examples/examples/data_io/query_http_csv.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::error::Result;
use datafusion::prelude::*;
use object_store::http::HttpBuilder;
diff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rs
index e4898e19d4..3e1568fb41 100644
--- a/datafusion-examples/examples/data_io/remote_catalog.rs
+++ b/datafusion-examples/examples/data_io/remote_catalog.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
/// This example shows how to implement the DataFusion [`CatalogProvider`] API
/// for catalogs that are remote (require network access) and/or offer only
/// asynchronous APIs such as [Polaris], [Unity], and [Hive].
diff --git a/datafusion-examples/examples/flight/client.rs b/datafusion-examples/examples/flight/client.rs
index 031beea47d..484576975a 100644
--- a/datafusion-examples/examples/flight/client.rs
+++ b/datafusion-examples/examples/flight/client.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::collections::HashMap;
use std::sync::Arc;
use tonic::transport::Endpoint;
diff --git a/datafusion-examples/examples/flight/server.rs b/datafusion-examples/examples/flight/server.rs
index dc75287cf2..e4c05a13e8 100644
--- a/datafusion-examples/examples/flight/server.rs
+++ b/datafusion-examples/examples/flight/server.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::ipc::writer::{CompressionContext, DictionaryTracker, IpcDataGenerator};
use std::sync::Arc;
diff --git a/datafusion-examples/examples/flight/sql_server.rs b/datafusion-examples/examples/flight/sql_server.rs
index d86860f9d4..529296fac8 100644
--- a/datafusion-examples/examples/flight/sql_server.rs
+++ b/datafusion-examples/examples/flight/sql_server.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::array::{ArrayRef, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow::ipc::writer::IpcWriteOptions;
diff --git a/datafusion-examples/examples/query_planning/analyzer_rule.rs b/datafusion-examples/examples/query_planning/analyzer_rule.rs
index b6c97679cb..df1ea4c1d7 100644
--- a/datafusion-examples/examples/query_planning/analyzer_rule.rs
+++ b/datafusion-examples/examples/query_planning/analyzer_rule.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
use datafusion::common::config::ConfigOptions;
use datafusion::common::tree_node::{Transformed, TreeNode};
diff --git a/datafusion-examples/examples/query_planning/expr_api.rs b/datafusion-examples/examples/query_planning/expr_api.rs
index 236ac4319b..225be0ee13 100644
--- a/datafusion-examples/examples/query_planning/expr_api.rs
+++ b/datafusion-examples/examples/query_planning/expr_api.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::collections::HashMap;
use std::sync::Arc;
diff --git a/datafusion-examples/examples/query_planning/optimizer_rule.rs b/datafusion-examples/examples/query_planning/optimizer_rule.rs
index 4af5ef50b3..34aef85c64 100644
--- a/datafusion-examples/examples/query_planning/optimizer_rule.rs
+++ b/datafusion-examples/examples/query_planning/optimizer_rule.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
use arrow::datatypes::DataType;
use datafusion::common::tree_node::{Transformed, TreeNode};
diff --git a/datafusion-examples/examples/query_planning/parse_sql_expr.rs b/datafusion-examples/examples/query_planning/parse_sql_expr.rs
index 44e6b3cf5f..376120de9d 100644
--- a/datafusion-examples/examples/query_planning/parse_sql_expr.rs
+++ b/datafusion-examples/examples/query_planning/parse_sql_expr.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::datatypes::{DataType, Field, Schema};
use datafusion::common::DFSchema;
use datafusion::logical_expr::{col, lit};
diff --git a/datafusion-examples/examples/query_planning/plan_to_sql.rs b/datafusion-examples/examples/query_planning/plan_to_sql.rs
index f23f083acd..95e04907d9 100644
--- a/datafusion-examples/examples/query_planning/plan_to_sql.rs
+++ b/datafusion-examples/examples/query_planning/plan_to_sql.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::common::DFSchemaRef;
use datafusion::error::Result;
use datafusion::logical_expr::sqlparser::ast::Statement;
diff --git a/datafusion-examples/examples/query_planning/planner_api.rs b/datafusion-examples/examples/query_planning/planner_api.rs
index dd3643471e..9b8aa1c2fe 100644
--- a/datafusion-examples/examples/query_planning/planner_api.rs
+++ b/datafusion-examples/examples/query_planning/planner_api.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::error::Result;
use datafusion::logical_expr::LogicalPlan;
use datafusion::physical_plan::displayable;
diff --git a/datafusion-examples/examples/query_planning/pruning.rs b/datafusion-examples/examples/query_planning/pruning.rs
index 7c42d151c0..d7562139c2 100644
--- a/datafusion-examples/examples/query_planning/pruning.rs
+++ b/datafusion-examples/examples/query_planning/pruning.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::collections::HashSet;
use std::sync::Arc;
diff --git a/datafusion-examples/examples/query_planning/thread_pools.rs b/datafusion-examples/examples/query_planning/thread_pools.rs
index 768638d11a..6fc7d51e91 100644
--- a/datafusion-examples/examples/query_planning/thread_pools.rs
+++ b/datafusion-examples/examples/query_planning/thread_pools.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! This example shows how to use separate thread pools (tokio [`Runtime`]))s to
//! run the IO and CPU intensive parts of DataFusion plans.
//!
diff --git a/datafusion-examples/examples/sql_analysis.rs b/datafusion-examples/examples/sql_ops/analysis.rs
similarity index 98%
rename from datafusion-examples/examples/sql_analysis.rs
rename to datafusion-examples/examples/sql_ops/analysis.rs
index 4ff669faf1..266c650772 100644
--- a/datafusion-examples/examples/sql_analysis.rs
+++ b/datafusion-examples/examples/sql_ops/analysis.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! This example shows how to use the structures that DataFusion provides to perform
//! Analysis on SQL queries and their plans.
//!
@@ -32,141 +34,9 @@ use datafusion::{
};
use test_utils::tpcds::tpcds_schemas;
-/// Counts the total number of joins in a plan
-fn total_join_count(plan: &LogicalPlan) -> usize {
- let mut total = 0;
-
- // We can use the TreeNode API to walk over a LogicalPlan.
- plan.apply(|node| {
- // if we encounter a join we update the running count
- if matches!(node, LogicalPlan::Join(_)) {
- total += 1;
- }
- Ok(TreeNodeRecursion::Continue)
- })
- .unwrap();
-
- total
-}
-
-/// Counts the total number of joins in a plan and collects every join tree in
-/// the plan with their respective join count.
-///
-/// Join Tree Definition: the largest subtree consisting entirely of joins
-///
-/// For example, this plan:
-///
-/// ```text
-/// JOIN
-/// / \
-/// A JOIN
-/// / \
-/// B C
-/// ```
-///
-/// has a single join tree `(A-B-C)` which will result in `(2, [2])`
-///
-/// This plan:
-///
-/// ```text
-/// JOIN
-/// / \
-/// A GROUP
-/// |
-/// JOIN
-/// / \
-/// B C
-/// ```
-///
-/// Has two join trees `(A-, B-C)` which will result in `(2, [1, 1])`
-fn count_trees(plan: &LogicalPlan) -> (usize, Vec<usize>) {
- // this works the same way as `total_count`, but now when we encounter a Join
- // we try to collect it's entire tree
- let mut to_visit = vec![plan];
- let mut total = 0;
- let mut groups = vec![];
-
- while let Some(node) = to_visit.pop() {
- // if we encounter a join, we know were at the root of the tree
- // count this tree and recurse on it's inputs
- if matches!(node, LogicalPlan::Join(_)) {
- let (group_count, inputs) = count_tree(node);
- total += group_count;
- groups.push(group_count);
- to_visit.extend(inputs);
- } else {
- to_visit.extend(node.inputs());
- }
- }
-
- (total, groups)
-}
-
-/// Count the entire join tree and return its inputs using TreeNode API
-///
-/// For example, if this function receives following plan:
-///
-/// ```text
-/// JOIN
-/// / \
-/// A GROUP
-/// |
-/// JOIN
-/// / \
-/// B C
-/// ```
-///
-/// It will return `(1, [A, GROUP])`
-fn count_tree(join: &LogicalPlan) -> (usize, Vec<&LogicalPlan>) {
- let mut inputs = Vec::new();
- let mut total = 0;
-
- join.apply(|node| {
- // Some extra knowledge:
- //
- // optimized plans have their projections pushed down as far as
- // possible, which sometimes results in a projection going in between 2
- // subsequent joins giving the illusion these joins are not "related",
- // when in fact they are.
- //
- // This plan:
- // JOIN
- // / \
- // A PROJECTION
- // |
- // JOIN
- // / \
- // B C
- //
- // is the same as:
- //
- // JOIN
- // / \
- // A JOIN
- // / \
- // B C
- // we can continue the recursion in this case
- if let LogicalPlan::Projection(_) = node {
- return Ok(TreeNodeRecursion::Continue);
- }
-
- // any join we count
- if matches!(node, LogicalPlan::Join(_)) {
- total += 1;
- Ok(TreeNodeRecursion::Continue)
- } else {
- inputs.push(node);
- // skip children of input node
- Ok(TreeNodeRecursion::Jump)
- }
- })
- .unwrap();
-
- (total, inputs)
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
+/// Demonstrates how to analyze a SQL query by counting JOINs and identifying
+/// join-trees using DataFusion’s `LogicalPlan` and `TreeNode` API.
+pub async fn analysis() -> Result<()> {
// To show how we can count the joins in a sql query we'll be using query 88
// from the TPC-DS benchmark.
//
@@ -310,3 +180,136 @@ from
Ok(())
}
+
+/// Counts the total number of joins in a plan
+fn total_join_count(plan: &LogicalPlan) -> usize {
+ let mut total = 0;
+
+ // We can use the TreeNode API to walk over a LogicalPlan.
+ plan.apply(|node| {
+ // if we encounter a join we update the running count
+ if matches!(node, LogicalPlan::Join(_)) {
+ total += 1;
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })
+ .unwrap();
+
+ total
+}
+
+/// Counts the total number of joins in a plan and collects every join tree in
+/// the plan with their respective join count.
+///
+/// Join Tree Definition: the largest subtree consisting entirely of joins
+///
+/// For example, this plan:
+///
+/// ```text
+/// JOIN
+/// / \
+/// A JOIN
+/// / \
+/// B C
+/// ```
+///
+/// has a single join tree `(A-B-C)` which will result in `(2, [2])`
+///
+/// This plan:
+///
+/// ```text
+/// JOIN
+/// / \
+/// A GROUP
+/// |
+/// JOIN
+/// / \
+/// B C
+/// ```
+///
+/// Has two join trees `(A-, B-C)` which will result in `(2, [1, 1])`
+fn count_trees(plan: &LogicalPlan) -> (usize, Vec<usize>) {
+ // this works the same way as `total_count`, but now when we encounter a Join
+ // we try to collect it's entire tree
+ let mut to_visit = vec![plan];
+ let mut total = 0;
+ let mut groups = vec![];
+
+ while let Some(node) = to_visit.pop() {
+ // if we encounter a join, we know were at the root of the tree
+ // count this tree and recurse on it's inputs
+ if matches!(node, LogicalPlan::Join(_)) {
+ let (group_count, inputs) = count_tree(node);
+ total += group_count;
+ groups.push(group_count);
+ to_visit.extend(inputs);
+ } else {
+ to_visit.extend(node.inputs());
+ }
+ }
+
+ (total, groups)
+}
+
+/// Count the entire join tree and return its inputs using TreeNode API
+///
+/// For example, if this function receives following plan:
+///
+/// ```text
+/// JOIN
+/// / \
+/// A GROUP
+/// |
+/// JOIN
+/// / \
+/// B C
+/// ```
+///
+/// It will return `(1, [A, GROUP])`
+fn count_tree(join: &LogicalPlan) -> (usize, Vec<&LogicalPlan>) {
+ let mut inputs = Vec::new();
+ let mut total = 0;
+
+ join.apply(|node| {
+ // Some extra knowledge:
+ //
+ // optimized plans have their projections pushed down as far as
+ // possible, which sometimes results in a projection going in between 2
+ // subsequent joins giving the illusion these joins are not "related",
+ // when in fact they are.
+ //
+ // This plan:
+ // JOIN
+ // / \
+ // A PROJECTION
+ // |
+ // JOIN
+ // / \
+ // B C
+ //
+ // is the same as:
+ //
+ // JOIN
+ // / \
+ // A JOIN
+ // / \
+ // B C
+ // we can continue the recursion in this case
+ if let LogicalPlan::Projection(_) = node {
+ return Ok(TreeNodeRecursion::Continue);
+ }
+
+ // any join we count
+ if matches!(node, LogicalPlan::Join(_)) {
+ total += 1;
+ Ok(TreeNodeRecursion::Continue)
+ } else {
+ inputs.push(node);
+ // skip children of input node
+ Ok(TreeNodeRecursion::Jump)
+ }
+ })
+ .unwrap();
+
+ (total, inputs)
+}
diff --git a/datafusion-examples/examples/sql_dialect.rs b/datafusion-examples/examples/sql_ops/dialect.rs
similarity index 98%
rename from datafusion-examples/examples/sql_dialect.rs
rename to datafusion-examples/examples/sql_ops/dialect.rs
index 20b515506f..62a9f3fa56 100644
--- a/datafusion-examples/examples/sql_dialect.rs
+++ b/datafusion-examples/examples/sql_ops/dialect.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::fmt::Display;
use datafusion::error::{DataFusionError, Result};
@@ -26,8 +28,7 @@ use datafusion::sql::{
/// This example demonstrates how to use the DFParser to parse a statement in a custom way
///
/// This technique can be used to implement a custom SQL dialect, for example.
-#[tokio::main]
-async fn main() -> Result<()> {
+pub async fn dialect() -> Result<()> {
let mut my_parser =
MyParser::new("COPY source_table TO 'file.fasta' STORED AS FASTA")?;
diff --git a/datafusion-examples/examples/sql_frontend.rs b/datafusion-examples/examples/sql_ops/frontend.rs
similarity index 99%
rename from datafusion-examples/examples/sql_frontend.rs
rename to datafusion-examples/examples/sql_ops/frontend.rs
index 1fc9ce24ec..2cfb9de05c 100644
--- a/datafusion-examples/examples/sql_frontend.rs
+++ b/datafusion-examples/examples/sql_ops/frontend.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::common::{plan_err, TableReference};
use datafusion::config::ConfigOptions;
@@ -44,7 +46,7 @@ use std::sync::Arc;
///
/// In this example, we demonstrate how to use the lower level APIs directly,
/// which only requires the `datafusion-sql` dependency.
-pub fn main() -> Result<()> {
+pub fn frontend() -> Result<()> {
// First, we parse the SQL string. Note that we use the DataFusion
// Parser, which wraps the `sqlparser-rs` SQL parser and adds DataFusion
// specific syntax such as `CREATE EXTERNAL TABLE`
diff --git a/datafusion-examples/examples/sql_ops/main.rs b/datafusion-examples/examples/sql_ops/main.rs
new file mode 100644
index 0000000000..d67b16bf5c
--- /dev/null
+++ b/datafusion-examples/examples/sql_ops/main.rs
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! # SQL Examples
+//!
+//! These examples demonstrate SQL operations in DataFusion.
+//!
+//! ## Usage
+//! ```bash
+//! cargo run --example sql_ops -- [analysis|dialect|frontend|query]
+//! ```
+//!
+//! Each subcommand runs a corresponding example:
+//! - `analysis` — analyse SQL queries with DataFusion structures
+//! - `dialect` — implementing a custom SQL dialect on top of DFParser
+//! - `frontend` — create LogicalPlans (only) from sql strings
+//! - `query` — query data using SQL (in memory RecordBatches, local Parquet files)
+
+mod analysis;
+mod dialect;
+mod frontend;
+mod query;
+
+use std::str::FromStr;
+
+use datafusion::error::{DataFusionError, Result};
+
+enum ExampleKind {
+ Analysis,
+ Dialect,
+ Frontend,
+ Query,
+}
+
+impl AsRef<str> for ExampleKind {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::Analysis => "analysis",
+ Self::Dialect => "dialect",
+ Self::Frontend => "frontend",
+ Self::Query => "query",
+ }
+ }
+}
+
+impl FromStr for ExampleKind {
+ type Err = DataFusionError;
+
+ fn from_str(s: &str) -> Result<Self> {
+ match s {
+ "analysis" => Ok(Self::Analysis),
+ "dialect" => Ok(Self::Dialect),
+ "frontend" => Ok(Self::Frontend),
+ "query" => Ok(Self::Query),
+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),
+ }
+ }
+}
+
+impl ExampleKind {
+ const ALL: [Self; 4] = [Self::Analysis, Self::Dialect, Self::Frontend, Self::Query];
+
+ const EXAMPLE_NAME: &str = "sql_ops";
+
+ fn variants() -> Vec<&'static str> {
+ Self::ALL.iter().map(|x| x.as_ref()).collect()
+ }
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let usage = format!(
+ "Usage: cargo run --example {} -- [{}]",
+ ExampleKind::EXAMPLE_NAME,
+ ExampleKind::variants().join("|")
+ );
+
+ let arg = std::env::args().nth(1).ok_or_else(|| {
+ eprintln!("{usage}");
+ DataFusionError::Execution("Missing argument".to_string())
+ })?;
+
+ match arg.parse::<ExampleKind>()? {
+ ExampleKind::Analysis => analysis::analysis().await?,
+ ExampleKind::Dialect => dialect::dialect().await?,
+ ExampleKind::Frontend => frontend::frontend()?,
+ ExampleKind::Query => query::query().await?,
+ }
+
+ Ok(())
+}
diff --git a/datafusion-examples/examples/sql_query.rs b/datafusion-examples/examples/sql_ops/query.rs
similarity index 99%
rename from datafusion-examples/examples/sql_query.rs
rename to datafusion-examples/examples/sql_ops/query.rs
index 4da07d33d0..3e052f0823 100644
--- a/datafusion-examples/examples/sql_query.rs
+++ b/datafusion-examples/examples/sql_ops/query.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::arrow::array::{UInt64Array, UInt8Array};
use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::arrow::record_batch::RecordBatch;
@@ -32,8 +34,7 @@ use std::sync::Arc;
///
/// [`query_memtable`]: a simple query against a [`MemTable`]
/// [`query_parquet`]: a simple query against a directory with multiple Parquet files
-#[tokio::main]
-async fn main() -> Result<()> {
+pub async fn query() -> Result<()> {
query_memtable().await?;
query_parquet().await?;
Ok(())
diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rs
index 81e227bfac..b4916e5bd0 100644
--- a/datafusion-examples/examples/udf/advanced_udaf.rs
+++ b/datafusion-examples/examples/udf/advanced_udaf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::datatypes::{Field, Schema};
use datafusion::physical_expr::NullState;
use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility};
diff --git a/datafusion-examples/examples/udf/advanced_udf.rs b/datafusion-examples/examples/udf/advanced_udf.rs
index bb5a68e90c..7459501a1c 100644
--- a/datafusion-examples/examples/udf/advanced_udf.rs
+++ b/datafusion-examples/examples/udf/advanced_udf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::any::Any;
use std::sync::Arc;
diff --git a/datafusion-examples/examples/udf/advanced_udwf.rs b/datafusion-examples/examples/udf/advanced_udwf.rs
index 86f215e019..37b6671639 100644
--- a/datafusion-examples/examples/udf/advanced_udwf.rs
+++ b/datafusion-examples/examples/udf/advanced_udwf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::datatypes::Field;
use arrow::{
array::{ArrayRef, AsArray, Float64Array},
diff --git a/datafusion-examples/examples/udf/async_udf.rs b/datafusion-examples/examples/udf/async_udf.rs
index 475775a599..c55650223c 100644
--- a/datafusion-examples/examples/udf/async_udf.rs
+++ b/datafusion-examples/examples/udf/async_udf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! This example shows how to create and use "Async UDFs" in DataFusion.
//!
//! Async UDFs allow you to perform asynchronous operations, such as
diff --git a/datafusion-examples/examples/udf/simple_udaf.rs b/datafusion-examples/examples/udf/simple_udaf.rs
index e9f905e720..42ea0054b7 100644
--- a/datafusion-examples/examples/udf/simple_udaf.rs
+++ b/datafusion-examples/examples/udf/simple_udaf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
/// In this example we will declare a single-type, single return type UDAF that computes the geometric mean.
/// The geometric mean is described here: https://en.wikipedia.org/wiki/Geometric_mean
use datafusion::arrow::{
diff --git a/datafusion-examples/examples/udf/simple_udf.rs b/datafusion-examples/examples/udf/simple_udf.rs
index 7d4f3588e3..e8d6c9c817 100644
--- a/datafusion-examples/examples/udf/simple_udf.rs
+++ b/datafusion-examples/examples/udf/simple_udf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use datafusion::{
arrow::{
array::{ArrayRef, Float32Array, Float64Array},
diff --git a/datafusion-examples/examples/udf/simple_udtf.rs b/datafusion-examples/examples/udf/simple_udtf.rs
index a03b157134..12ee74fc52 100644
--- a/datafusion-examples/examples/udf/simple_udtf.rs
+++ b/datafusion-examples/examples/udf/simple_udtf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use arrow::csv::reader::Format;
use arrow::csv::ReaderBuilder;
use async_trait::async_trait;
diff --git a/datafusion-examples/examples/udf/simple_udwf.rs b/datafusion-examples/examples/udf/simple_udwf.rs
index 2cf1df8d8e..a4f6d59a7d 100644
--- a/datafusion-examples/examples/udf/simple_udwf.rs
+++ b/datafusion-examples/examples/udf/simple_udwf.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+
use std::sync::Arc;
use arrow::{