Apache DataFusion SQL Query Engine
Consolidate execution monitoring examples (#18142) (#18846)
Sergey Zhukov356616e0c47e
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 937fb779ec..5453480165 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -69,8 +69,9 @@ cargo run --example dataframe- [`examples/custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks.- [`flight/sql_server.rs`](examples/flight/sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from Flight and and FlightSQL (e.g. JDBC) clients- [`examples/builtin_functions/function_factory.rs`](examples/builtin_functions/function_factory.rs): Register `CREATE FUNCTION` handler to implement SQL macros-- [`memory_pool_tracking.rs`](examples/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages-- [`memory_pool_execution_plan.rs`](examples/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling+- [`examples/execution_monitoring/memory_pool_tracking.rs`](examples/execution_monitoring/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages+- [`examples/execution_monitoring/memory_pool_execution_plan.rs`](examples/execution_monitoring/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling+- [`examples/execution_monitoring/tracing.rs`](examples/execution_monitoring/tracing.rs): Demonstrates the tracing injection feature for the DataFusion runtime- [`examples/query_planning/optimizer_rule.rs`](examples/query_planning/optimizer_rule.rs): Use a custom OptimizerRule to replace certain predicates- [`examples/data_io/parquet_embedded_index.rs`](examples/data_io/parquet_embedded_index.rs): Store a custom index inside a Parquet file and use it to speed up queries- [`examples/data_io/parquet_encrypted.rs`](examples/data_io/parquet_encrypted.rs): Read and write encrypted Parquet files using DataFusiondiff --git a/datafusion-examples/examples/execution_monitoring/main.rs b/datafusion-examples/examples/execution_monitoring/main.rsnew file mode 100644index 0000000000..fd834cf7b7--- /dev/null+++ b/datafusion-examples/examples/execution_monitoring/main.rs@@ -0,0 +1,107 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! # These examples of memory and performance management+//!+//! These examples demonstrate memory and performance management.+//!+//! ## Usage+//! ```bash+//! cargo run --example execution_monitoring -- [mem_pool_exec_plan|mem_pool_tracking|tracing]+//! ```+//!+//! Each subcommand runs a corresponding example:+//! - `mem_pool_exec_plan` — shows how to implement memory-aware ExecutionPlan with memory reservation and spilling+//! - `mem_pool_tracking` — demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages+//! - `tracing` — demonstrates the tracing injection feature for the DataFusion runtime++mod memory_pool_execution_plan;+mod memory_pool_tracking;+mod tracing;++use std::str::FromStr;++use datafusion::error::{DataFusionError, Result};++enum ExampleKind {+ MemoryPoolExecutionPlan,+ MemoryPoolTracking,+ Tracing,+}++impl AsRef<str> for ExampleKind {+ fn as_ref(&self) -> &str {+ match self {+ Self::MemoryPoolExecutionPlan => "mem_pool_exec_plan",+ Self::MemoryPoolTracking => "mem_pool_tracking",+ Self::Tracing => "tracing",+ }+ }+}++impl FromStr for ExampleKind {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ "mem_pool_exec_plan" => Ok(Self::MemoryPoolExecutionPlan),+ "mem_pool_tracking" => Ok(Self::MemoryPoolTracking),+ "tracing" => Ok(Self::Tracing),+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),+ }+ }+}++impl ExampleKind {+ const ALL: [Self; 3] = [+ Self::MemoryPoolExecutionPlan,+ Self::MemoryPoolTracking,+ Self::Tracing,+ ];++ const EXAMPLE_NAME: &str = "execution_monitoring";++ fn variants() -> Vec<&'static str> {+ Self::ALL.iter().map(|x| x.as_ref()).collect()+ }+}++#[tokio::main]+async fn main() -> Result<()> {+ let usage = format!(+ "Usage: cargo run --example {} -- [{}]",+ ExampleKind::EXAMPLE_NAME,+ ExampleKind::variants().join("|")+ );++ let arg = std::env::args().nth(1).ok_or_else(|| {+ eprintln!("{usage}");+ DataFusionError::Execution("Missing argument".to_string())+ })?;++ match arg.parse::<ExampleKind>()? {+ ExampleKind::MemoryPoolExecutionPlan => {+ memory_pool_execution_plan::memory_pool_execution_plan().await?+ }+ ExampleKind::MemoryPoolTracking => {+ memory_pool_tracking::mem_pool_tracking().await?+ }+ ExampleKind::Tracing => tracing::tracing().await?,+ }++ Ok(())+}diff --git a/datafusion-examples/examples/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rssimilarity index 98%rename from datafusion-examples/examples/memory_pool_execution_plan.rsrename to datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rsindex 3258cde176..896d244fbd 100644--- a/datafusion-examples/examples/memory_pool_execution_plan.rs+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! This example demonstrates how to implement custom ExecutionPlans that properly//! use memory tracking through TrackConsumersPool.//!@@ -44,8 +46,8 @@ use std::any::Any;use std::fmt;use std::sync::Arc;-#[tokio::main]-async fn main() -> Result<(), Box<dyn std::error::Error>> {+/// Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling+pub async fn memory_pool_execution_plan() -> Result<()> {println!("=== DataFusion ExecutionPlan Memory Tracking Example ===\n");// Set up a runtime with memory trackingdiff --git a/datafusion-examples/examples/memory_pool_tracking.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rssimilarity index 95%rename from datafusion-examples/examples/memory_pool_tracking.rsrename to datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rsindex d5823b1173..8d6e5dd7e4 100644--- a/datafusion-examples/examples/memory_pool_tracking.rs+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_tracking.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! This example demonstrates how to use TrackConsumersPool for memory tracking and debugging.//!//! The TrackConsumersPool provides enhanced error messages that show the top memory consumers@@ -24,11 +26,12 @@//!//! * [`automatic_usage_example`]: Shows how to use RuntimeEnvBuilder to automatically enable memory tracking+use datafusion::error::Result;use datafusion::execution::runtime_env::RuntimeEnvBuilder;use datafusion::prelude::*;-#[tokio::main]-async fn main() -> Result<(), Box<dyn std::error::Error>> {+/// Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages+pub async fn mem_pool_tracking() -> Result<()> {println!("=== DataFusion Memory Pool Tracking Example ===\n");// Example 1: Automatic Usage with RuntimeEnvBuilder@@ -41,7 +44,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {////// This shows the recommended way to use TrackConsumersPool through RuntimeEnvBuilder,/// which automatically creates a TrackConsumersPool with sensible defaults.-async fn automatic_usage_example() -> datafusion::error::Result<()> {+async fn automatic_usage_example() -> Result<()> {println!("Example 1: Automatic Usage with RuntimeEnvBuilder");println!("------------------------------------------------");diff --git a/datafusion-examples/examples/tracing.rs b/datafusion-examples/examples/execution_monitoring/tracing.rssimilarity index 97%rename from datafusion-examples/examples/tracing.rsrename to datafusion-examples/examples/execution_monitoring/tracing.rsindex 334ee0f4e5..f0e5a03715 100644--- a/datafusion-examples/examples/tracing.rs+++ b/datafusion-examples/examples/execution_monitoring/tracing.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+//!//! This example demonstrates the tracing injection feature for the DataFusion runtime.//! Tasks spawned on new threads behave differently depending on whether a tracer is injected.//! The log output clearly distinguishes the two cases.@@ -61,8 +63,8 @@ use std::any::Any;use std::sync::Arc;use tracing::{info, instrument, Instrument, Level, Span};-#[tokio::main]-async fn main() -> Result<()> {+/// Demonstrates the tracing injection feature for the DataFusion runtime+pub async fn tracing() -> Result<()> {// Initialize tracing subscriber with thread info.tracing_subscriber::fmt().with_thread_ids(true)