Apache DataFusion SQL Query Engine
Consolidate flight examples (#18142) (#18442)
Sergey Zhukov2d5c10194364
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.tomlindex bb0525e577..0ec410ecc6 100644--- a/datafusion-examples/Cargo.toml+++ b/datafusion-examples/Cargo.toml@@ -32,18 +32,6 @@ rust-version = { workspace = true }[lints]workspace = true-[[example]]-name = "flight_sql_server"-path = "examples/flight/flight_sql_server.rs"--[[example]]-name = "flight_server"-path = "examples/flight/flight_server.rs"--[[example]]-name = "flight_client"-path = "examples/flight/flight_client.rs"-[[example]]name = "dataframe_to_s3"path = "examples/external_dependency/dataframe-to-s3.rs"diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex f1bcbcce82..f6783a643f 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -65,7 +65,7 @@ cargo run --example dataframe- [`deserialize_to_struct.rs`](examples/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs- [`expr_api.rs`](examples/expr_api.rs): Create, execute, simplify, analyze and coerce `Expr`s- [`file_stream_provider.rs`](examples/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks.-- [`flight_sql_server.rs`](examples/flight/flight_sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from JDBC clients+- [`flight/sql_server.rs`](examples/flight/sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from Flight and and FlightSQL (e.g. JDBC) clients- [`function_factory.rs`](examples/function_factory.rs): Register `CREATE FUNCTION` handler to implement SQL macros- [`memory_pool_tracking.rs`](examples/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages- [`memory_pool_execution_plan.rs`](examples/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling@@ -94,4 +94,4 @@ cargo run --example dataframe## Distributed-- [`flight_client.rs`](examples/flight/flight_client.rs) and [`flight_server.rs`](examples/flight/flight_server.rs): Run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol.+- [`examples/flight/client.rs`](examples/flight/client.rs) and [`examples/flight/server.rs`](examples/flight/server.rs): Run DataFusion as a standalone process and execute SQL queries from a client using the Arrow Flight protocol.diff --git a/datafusion-examples/examples/flight/flight_client.rs b/datafusion-examples/examples/flight/client.rssimilarity index 97%rename from datafusion-examples/examples/flight/flight_client.rsrename to datafusion-examples/examples/flight/client.rsindex ff4b5903ad..031beea47d 100644--- a/datafusion-examples/examples/flight/flight_client.rs+++ b/datafusion-examples/examples/flight/client.rs@@ -30,8 +30,7 @@ use datafusion::arrow::util::pretty;/// This example shows how to wrap DataFusion with `FlightService` to support looking up schema information for/// Parquet files and executing SQL queries against them on a remote server./// This example is run along-side the example `flight_server`.-#[tokio::main]-async fn main() -> Result<(), Box<dyn std::error::Error>> {+pub async fn client() -> Result<(), Box<dyn std::error::Error>> {let testdata = datafusion::test_util::parquet_test_data();// Create Flight clientdiff --git a/datafusion-examples/examples/flight/main.rs b/datafusion-examples/examples/flight/main.rsnew file mode 100644index 0000000000..a448789b35--- /dev/null+++ b/datafusion-examples/examples/flight/main.rs@@ -0,0 +1,94 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! # Arrow Flight Examples+//!+//! These examples demonstrate Arrow Flight usage.+//!+//! Each subcommand runs a corresponding example:+//! - `client` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol+//! - `server` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol+//! - `sql_server` — run DataFusion as a standalone process and execute SQL queries from JDBC clients++mod client;+mod server;+mod sql_server;++use std::str::FromStr;++use datafusion::error::{DataFusionError, Result};++enum ExampleKind {+ Client,+ Server,+ SqlServer,+}++impl AsRef<str> for ExampleKind {+ fn as_ref(&self) -> &str {+ match self {+ Self::Client => "client",+ Self::Server => "server",+ Self::SqlServer => "sql_server",+ }+ }+}++impl FromStr for ExampleKind {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ "client" => Ok(Self::Client),+ "server" => Ok(Self::Server),+ "sql_server" => Ok(Self::SqlServer),+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),+ }+ }+}++impl ExampleKind {+ const ALL: [Self; 3] = [Self::Client, Self::Server, Self::SqlServer];++ const EXAMPLE_NAME: &str = "flight";++ fn variants() -> Vec<&'static str> {+ Self::ALL.iter().map(|x| x.as_ref()).collect()+ }+}++#[tokio::main]+async fn main() -> Result<(), Box<dyn std::error::Error>> {+ let usage = format!(+ "Usage: cargo run --example {} -- [{}]",+ ExampleKind::EXAMPLE_NAME,+ ExampleKind::variants().join("|")+ );++ let arg = std::env::args().nth(1).ok_or_else(|| {+ eprintln!("{usage}");+ DataFusionError::Execution("Missing argument".to_string())+ })?;++ match arg.parse::<ExampleKind>()? {+ ExampleKind::Client => client::client().await?,+ ExampleKind::Server => server::server().await?,+ ExampleKind::SqlServer => sql_server::sql_server().await?,+ }++ Ok(())+}diff --git a/datafusion-examples/examples/flight/flight_server.rs b/datafusion-examples/examples/flight/server.rssimilarity index 99%rename from datafusion-examples/examples/flight/flight_server.rsrename to datafusion-examples/examples/flight/server.rsindex 22265e415f..dc75287cf2 100644--- a/datafusion-examples/examples/flight/flight_server.rs+++ b/datafusion-examples/examples/flight/server.rs@@ -194,8 +194,7 @@ fn to_tonic_err(e: datafusion::error::DataFusionError) -> Status {/// This example shows how to wrap DataFusion with `FlightService` to support looking up schema information for/// Parquet files and executing SQL queries against them on a remote server./// This example is run along-side the example `flight_client`.-#[tokio::main]-async fn main() -> Result<(), Box<dyn std::error::Error>> {+pub async fn server() -> Result<(), Box<dyn std::error::Error>> {let addr = "0.0.0.0:50051".parse()?;let service = FlightServiceImpl {};diff --git a/datafusion-examples/examples/flight/flight_sql_server.rs b/datafusion-examples/examples/flight/sql_server.rssimilarity index 99%rename from datafusion-examples/examples/flight/flight_sql_server.rsrename to datafusion-examples/examples/flight/sql_server.rsindex c35debec7d..fc7d0817bd 100644--- a/datafusion-examples/examples/flight/flight_sql_server.rs+++ b/datafusion-examples/examples/flight/sql_server.rs@@ -69,8 +69,7 @@ macro_rules! status {/// Based heavily on Ballista's implementation: https://github.com/apache/datafusion-ballista/blob/main/ballista/scheduler/src/flight_sql.rs/// and the example in arrow-rs: https://github.com/apache/arrow-rs/blob/master/arrow-flight/examples/flight_sql_server.rs///-#[tokio::main]-async fn main() -> Result<(), Box<dyn std::error::Error>> {+pub async fn sql_server() -> Result<(), Box<dyn std::error::Error>> {env_logger::init();let addr = "0.0.0.0:50051".parse()?;let service = FlightSqlServiceImpl {