Skip to content

Apache DataFusion SQL Query Engine

Consolidate execution monitoring examples (#18142) (#18846)

Sergey Zhukov356616e0c47e
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
index 937fb779ec..5453480165 100644
--- a/datafusion-examples/README.md
+++ b/datafusion-examples/README.md
@@ -69,8 +69,9 @@ cargo run --example dataframe
- [`examples/custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks.
- [`flight/sql_server.rs`](examples/flight/sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from Flight and and FlightSQL (e.g. JDBC) clients
- [`examples/builtin_functions/function_factory.rs`](examples/builtin_functions/function_factory.rs): Register `CREATE FUNCTION` handler to implement SQL macros
-- [`memory_pool_tracking.rs`](examples/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages
-- [`memory_pool_execution_plan.rs`](examples/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling
+- [`examples/execution_monitoring/memory_pool_tracking.rs`](examples/execution_monitoring/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages
+- [`examples/execution_monitoring/memory_pool_execution_plan.rs`](examples/execution_monitoring/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling
+- [`examples/execution_monitoring/tracing.rs`](examples/execution_monitoring/tracing.rs): Demonstrates the tracing injection feature for the DataFusion runtime
- [`examples/query_planning/optimizer_rule.rs`](examples/query_planning/optimizer_rule.rs): Use a custom OptimizerRule to replace certain predicates
- [`examples/data_io/parquet_embedded_index.rs`](examples/data_io/parquet_embedded_index.rs): Store a custom index inside a Parquet file and use it to speed up queries
- [`examples/data_io/parquet_encrypted.rs`](examples/data_io/parquet_encrypted.rs): Read and write encrypted Parquet files using DataFusion
diff --git a/datafusion-examples/examples/execution_monitoring/main.rs b/datafusion-examples/examples/execution_monitoring/main.rs
new file mode 100644
index 0000000000..fd834cf7b7
--- /dev/null
+++ b/datafusion-examples/examples/execution_monitoring/main.rs
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! # These examples of memory and performance management
+//!
+//! These examples demonstrate memory and performance management.
+//!
+//! ## Usage
+//! ```bash
+//! cargo run --example execution_monitoring -- [mem_pool_exec_plan|mem_pool_tracking|tracing]
+//! ```
+//!
+//! Each subcommand runs a corresponding example:
+//! - `mem_pool_exec_plan` — shows how to implement memory-aware ExecutionPlan with memory reservation and spilling
+//! - `mem_pool_tracking` — demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages
+//! - `tracing` — demonstrates the tracing injection feature for the DataFusion runtime
+
+mod memory_pool_execution_plan;
+mod memory_pool_tracking;
+mod tracing;
+
+use std::str::FromStr;
+
+use datafusion::error::{DataFusionError, Result};
+
+enum ExampleKind {
+ MemoryPoolExecutionPlan,
+ MemoryPoolTracking,
+ Tracing,
+}
+
+impl AsRef<str> for ExampleKind {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::MemoryPoolExecutionPlan => "mem_pool_exec_plan",
+ Self::MemoryPoolTracking => "mem_pool_tracking",
+ Self::Tracing => "tracing",
+ }
+ }
+}
+
+impl FromStr for ExampleKind {
+ type Err = DataFusionError;
+
+ fn from_str(s: &str) -> Result<Self> {
+ match s {
+ "mem_pool_exec_plan" => Ok(Self::MemoryPoolExecutionPlan),
+ "mem_pool_tracking" => Ok(Self::MemoryPoolTracking),
+ "tracing" => Ok(Self::Tracing),
+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),
+ }
+ }
+}
+
+impl ExampleKind {
+ const ALL: [Self; 3] = [
+ Self::MemoryPoolExecutionPlan,
+ Self::MemoryPoolTracking,
+ Self::Tracing,
+ ];
+
+ const EXAMPLE_NAME: &str = "execution_monitoring";
+
+ fn variants() -> Vec<&'static str> {
+ Self::ALL.iter().map(|x| x.as_ref()).collect()
+ }
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let usage = format!(
+ "Usage: cargo run --example {} -- [{}]",
+ ExampleKind::EXAMPLE_NAME,
+ ExampleKind::variants().join("|")
+ );
+
+ let arg = std::env::args().nth(1).ok_or_else(|| {
+ eprintln!("{usage}");
+ DataFusionError::Execution("Missing argument".to_string())
+ })?;
+
+ match arg.parse::<ExampleKind>()? {
+ ExampleKind::MemoryPoolExecutionPlan => {
+ memory_pool_execution_plan::memory_pool_execution_plan().await?
+ }
+ ExampleKind::MemoryPoolTracking => {
+ memory_pool_tracking::mem_pool_tracking().await?
+ }
+ ExampleKind::Tracing => tracing::tracing().await?,
+ }
+
+ Ok(())
+}
diff --git a/datafusion-examples/examples/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
similarity index 98%
rename from datafusion-examples/examples/memory_pool_execution_plan.rs
rename to datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
index 3258cde176..896d244fbd 100644
--- a/datafusion-examples/examples/memory_pool_execution_plan.rs
+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! This example demonstrates how to implement custom ExecutionPlans that properly
//! use memory tracking through TrackConsumersPool.
//!
@@ -44,8 +46,8 @@ use std::any::Any;
use std::fmt;
use std::sync::Arc;
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling
+pub async fn memory_pool_execution_plan() -> Result<()> {
println!("=== DataFusion ExecutionPlan Memory Tracking Example ===\n");
// Set up a runtime with memory tracking
diff --git a/datafusion-examples/examples/memory_pool_tracking.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs
similarity index 95%
rename from datafusion-examples/examples/memory_pool_tracking.rs
rename to datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs
index d5823b1173..8d6e5dd7e4 100644
--- a/datafusion-examples/examples/memory_pool_tracking.rs
+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! This example demonstrates how to use TrackConsumersPool for memory tracking and debugging.
//!
//! The TrackConsumersPool provides enhanced error messages that show the top memory consumers
@@ -24,11 +26,12 @@
//!
//! * [`automatic_usage_example`]: Shows how to use RuntimeEnvBuilder to automatically enable memory tracking
+use datafusion::error::Result;
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::prelude::*;
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages
+pub async fn mem_pool_tracking() -> Result<()> {
println!("=== DataFusion Memory Pool Tracking Example ===\n");
// Example 1: Automatic Usage with RuntimeEnvBuilder
@@ -41,7 +44,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
///
/// This shows the recommended way to use TrackConsumersPool through RuntimeEnvBuilder,
/// which automatically creates a TrackConsumersPool with sensible defaults.
-async fn automatic_usage_example() -> datafusion::error::Result<()> {
+async fn automatic_usage_example() -> Result<()> {
println!("Example 1: Automatic Usage with RuntimeEnvBuilder");
println!("------------------------------------------------");
diff --git a/datafusion-examples/examples/tracing.rs b/datafusion-examples/examples/execution_monitoring/tracing.rs
similarity index 97%
rename from datafusion-examples/examples/tracing.rs
rename to datafusion-examples/examples/execution_monitoring/tracing.rs
index 334ee0f4e5..f0e5a03715 100644
--- a/datafusion-examples/examples/tracing.rs
+++ b/datafusion-examples/examples/execution_monitoring/tracing.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+//! See `main.rs` for how to run it.
+//!
//! This example demonstrates the tracing injection feature for the DataFusion runtime.
//! Tasks spawned on new threads behave differently depending on whether a tracer is injected.
//! The log output clearly distinguishes the two cases.
@@ -61,8 +63,8 @@ use std::any::Any;
use std::sync::Arc;
use tracing::{info, instrument, Instrument, Level, Span};
-#[tokio::main]
-async fn main() -> Result<()> {
+/// Demonstrates the tracing injection feature for the DataFusion runtime
+pub async fn tracing() -> Result<()> {
// Initialize tracing subscriber with thread info.
tracing_subscriber::fmt()
.with_thread_ids(true)