Skip to content

Apache DataFusion SQL Query Engine

Consolidate flight examples (#18142) (#18442)

Sergey Zhukov2d5c10194364
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index bb0525e577..0ec410ecc6 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -32,18 +32,6 @@ rust-version = { workspace = true }
[lints]
workspace = true
-[[example]]
-name = "flight_sql_server"
-path = "examples/flight/flight_sql_server.rs"
-
-[[example]]
-name = "flight_server"
-path = "examples/flight/flight_server.rs"
-
-[[example]]
-name = "flight_client"
-path = "examples/flight/flight_client.rs"
-
[[example]]
name = "dataframe_to_s3"
path = "examples/external_dependency/dataframe-to-s3.rs"
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
index f1bcbcce82..f6783a643f 100644
--- a/datafusion-examples/README.md
+++ b/datafusion-examples/README.md
@@ -65,7 +65,7 @@ cargo run --example dataframe
- [`deserialize_to_struct.rs`](examples/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs
- [`expr_api.rs`](examples/expr_api.rs): Create, execute, simplify, analyze and coerce `Expr`s
- [`file_stream_provider.rs`](examples/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks.
-- [`flight_sql_server.rs`](examples/flight/flight_sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from JDBC clients
+- [`flight/sql_server.rs`](examples/flight/sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from Flight and and FlightSQL (e.g. JDBC) clients
- [`function_factory.rs`](examples/function_factory.rs): Register `CREATE FUNCTION` handler to implement SQL macros
- [`memory_pool_tracking.rs`](examples/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages
- [`memory_pool_execution_plan.rs`](examples/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling
@@ -94,4 +94,4 @@ cargo run --example dataframe
## Distributed
-- [`flight_client.rs`](examples/flight/flight_client.rs) and [`flight_server.rs`](examples/flight/flight_server.rs): Run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol.
+- [`examples/flight/client.rs`](examples/flight/client.rs) and [`examples/flight/server.rs`](examples/flight/server.rs): Run DataFusion as a standalone process and execute SQL queries from a client using the Arrow Flight protocol.
diff --git a/datafusion-examples/examples/flight/flight_client.rs b/datafusion-examples/examples/flight/client.rs
similarity index 97%
rename from datafusion-examples/examples/flight/flight_client.rs
rename to datafusion-examples/examples/flight/client.rs
index ff4b5903ad..031beea47d 100644
--- a/datafusion-examples/examples/flight/flight_client.rs
+++ b/datafusion-examples/examples/flight/client.rs
@@ -30,8 +30,7 @@ use datafusion::arrow::util::pretty;
/// This example shows how to wrap DataFusion with `FlightService` to support looking up schema information for
/// Parquet files and executing SQL queries against them on a remote server.
/// This example is run along-side the example `flight_server`.
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+pub async fn client() -> Result<(), Box<dyn std::error::Error>> {
let testdata = datafusion::test_util::parquet_test_data();
// Create Flight client
diff --git a/datafusion-examples/examples/flight/main.rs b/datafusion-examples/examples/flight/main.rs
new file mode 100644
index 0000000000..a448789b35
--- /dev/null
+++ b/datafusion-examples/examples/flight/main.rs
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! # Arrow Flight Examples
+//!
+//! These examples demonstrate Arrow Flight usage.
+//!
+//! Each subcommand runs a corresponding example:
+//! - `client` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol
+//! - `server` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol
+//! - `sql_server` — run DataFusion as a standalone process and execute SQL queries from JDBC clients
+
+mod client;
+mod server;
+mod sql_server;
+
+use std::str::FromStr;
+
+use datafusion::error::{DataFusionError, Result};
+
+enum ExampleKind {
+ Client,
+ Server,
+ SqlServer,
+}
+
+impl AsRef<str> for ExampleKind {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::Client => "client",
+ Self::Server => "server",
+ Self::SqlServer => "sql_server",
+ }
+ }
+}
+
+impl FromStr for ExampleKind {
+ type Err = DataFusionError;
+
+ fn from_str(s: &str) -> Result<Self> {
+ match s {
+ "client" => Ok(Self::Client),
+ "server" => Ok(Self::Server),
+ "sql_server" => Ok(Self::SqlServer),
+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),
+ }
+ }
+}
+
+impl ExampleKind {
+ const ALL: [Self; 3] = [Self::Client, Self::Server, Self::SqlServer];
+
+ const EXAMPLE_NAME: &str = "flight";
+
+ fn variants() -> Vec<&'static str> {
+ Self::ALL.iter().map(|x| x.as_ref()).collect()
+ }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let usage = format!(
+ "Usage: cargo run --example {} -- [{}]",
+ ExampleKind::EXAMPLE_NAME,
+ ExampleKind::variants().join("|")
+ );
+
+ let arg = std::env::args().nth(1).ok_or_else(|| {
+ eprintln!("{usage}");
+ DataFusionError::Execution("Missing argument".to_string())
+ })?;
+
+ match arg.parse::<ExampleKind>()? {
+ ExampleKind::Client => client::client().await?,
+ ExampleKind::Server => server::server().await?,
+ ExampleKind::SqlServer => sql_server::sql_server().await?,
+ }
+
+ Ok(())
+}
diff --git a/datafusion-examples/examples/flight/flight_server.rs b/datafusion-examples/examples/flight/server.rs
similarity index 99%
rename from datafusion-examples/examples/flight/flight_server.rs
rename to datafusion-examples/examples/flight/server.rs
index 22265e415f..dc75287cf2 100644
--- a/datafusion-examples/examples/flight/flight_server.rs
+++ b/datafusion-examples/examples/flight/server.rs
@@ -194,8 +194,7 @@ fn to_tonic_err(e: datafusion::error::DataFusionError) -> Status {
/// This example shows how to wrap DataFusion with `FlightService` to support looking up schema information for
/// Parquet files and executing SQL queries against them on a remote server.
/// This example is run along-side the example `flight_client`.
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+pub async fn server() -> Result<(), Box<dyn std::error::Error>> {
let addr = "0.0.0.0:50051".parse()?;
let service = FlightServiceImpl {};
diff --git a/datafusion-examples/examples/flight/flight_sql_server.rs b/datafusion-examples/examples/flight/sql_server.rs
similarity index 99%
rename from datafusion-examples/examples/flight/flight_sql_server.rs
rename to datafusion-examples/examples/flight/sql_server.rs
index c35debec7d..fc7d0817bd 100644
--- a/datafusion-examples/examples/flight/flight_sql_server.rs
+++ b/datafusion-examples/examples/flight/sql_server.rs
@@ -69,8 +69,7 @@ macro_rules! status {
/// Based heavily on Ballista's implementation: https://github.com/apache/datafusion-ballista/blob/main/ballista/scheduler/src/flight_sql.rs
/// and the example in arrow-rs: https://github.com/apache/arrow-rs/blob/master/arrow-flight/examples/flight_sql_server.rs
///
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+pub async fn sql_server() -> Result<(), Box<dyn std::error::Error>> {
env_logger::init();
let addr = "0.0.0.0:50051".parse()?;
let service = FlightSqlServiceImpl {