Apache DataFusion SQL Query Engine
Consolidate dataframe examples (#18142) (#18862)
Sergey Zhukovd3ae1a78556d
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 03c382b7cf..1595c69628 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -41,7 +41,7 @@ cd datafusion-examples/examples# Run the `dataframe` example:# ... use the equivalent for other examples-cargo run --example dataframe+cargo run --example dataframe -- dataframe```## Single Process@@ -61,10 +61,10 @@ cargo run --example dataframe- [`examples/custom_data_source/custom_file_casts.rs`](examples/custom_data_source/custom_file_casts.rs): Implement custom casting rules to adapt file schemas- [`examples/custom_data_source/custom_file_format.rs`](examples/custom_data_source/custom_file_format.rs): Write data to a custom file format- [`dataframe-to-s3.rs`](examples/external_dependency/dataframe-to-s3.rs): Run a query using a DataFrame against a parquet file from s3 and writing back to s3-- [`dataframe.rs`](examples/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries. Also demonstrates the various methods to write out a DataFrame to a table, parquet file, csv file, and json file.+- [`examples/dataframe/dataframe.rs`](examples/dataframe/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries. Also demonstrates the various methods to write out a DataFrame to a table, parquet file, csv file, and json file.- [`examples/builtin_functions/date_time`](examples/builtin_functions/date_time.rs): Examples of date-time related functions and queries-- [`default_column_values.rs`](examples/default_column_values.rs): Implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter-- [`deserialize_to_struct.rs`](examples/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs+- [`examples/custom_data_source/default_column_values.rs`](examples/custom_data_source/default_column_values.rs): Implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter+- [`examples/dataframe/deserialize_to_struct.rs`](examples/dataframe/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs- [`examples/query_planning/expr_api.rs`](examples/query_planning/expr_api.rs): Create, execute, simplify, analyze and coerce `Expr`s- [`examples/custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks.- [`flight/sql_server.rs`](examples/flight/sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from Flight and and FlightSQL (e.g. JDBC) clientsdiff --git a/datafusion-examples/examples/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rssimilarity index 92%rename from datafusion-examples/examples/default_column_values.rsrename to datafusion-examples/examples/custom_data_source/default_column_values.rsindex bfc60519f2..19d00e0a0d 100644--- a/datafusion-examples/examples/default_column_values.rs+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use std::any::Any;use std::collections::HashMap;use std::sync::Arc;@@ -52,25 +54,23 @@ use object_store::{ObjectStore, PutPayload};// Metadata key for storing default values in field metadataconst DEFAULT_VALUE_METADATA_KEY: &str = "example.default_value";-// Example showing how to implement custom default value handling for missing columns-// using field metadata and PhysicalExprAdapter.-//-// This example demonstrates how to:-// 1. Store default values in field metadata using a constant key-// 2. Create a custom PhysicalExprAdapter that reads these defaults-// 3. Inject default values for missing columns in filter predicates-// 4. Use the DefaultPhysicalExprAdapter as a fallback for standard schema adaptation-// 5. Wrap string default values in cast expressions for proper type conversion-//-// Important: PhysicalExprAdapter is specifically designed for rewriting filter predicates-// that get pushed down to file scans. For handling missing columns in projections,-// other mechanisms in DataFusion are used (like SchemaAdapter).-//-// The metadata-based approach provides a flexible way to store default values as strings-// and cast them to the appropriate types at query time.--#[tokio::main]-async fn main() -> Result<()> {+/// Example showing how to implement custom default value handling for missing columns+/// using field metadata and PhysicalExprAdapter.+///+/// This example demonstrates how to:+/// 1. Store default values in field metadata using a constant key+/// 2. Create a custom PhysicalExprAdapter that reads these defaults+/// 3. Inject default values for missing columns in filter predicates+/// 4. Use the DefaultPhysicalExprAdapter as a fallback for standard schema adaptation+/// 5. Wrap string default values in cast expressions for proper type conversion+///+/// Important: PhysicalExprAdapter is specifically designed for rewriting filter predicates+/// that get pushed down to file scans. For handling missing columns in projections,+/// other mechanisms in DataFusion are used (like SchemaAdapter).+///+/// The metadata-based approach provides a flexible way to store default values as strings+/// and cast them to the appropriate types at query time.+pub async fn default_column_values() -> Result<()> {println!("=== Creating example data with missing columns and default values ===");// Create sample data where the logical schema has more columns than the physical schema@@ -85,11 +85,10 @@ async fn main() -> Result<()> {.build();let mut writer =- ArrowWriter::try_new(&mut buf, physical_schema.clone(), Some(props))- .expect("creating writer");+ ArrowWriter::try_new(&mut buf, physical_schema.clone(), Some(props))?;- writer.write(&batch).expect("Writing batch");- writer.close().unwrap();+ writer.write(&batch)?;+ writer.close()?;buf};let path = Path::from("example.parquet");diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rsindex ce0585f8c3..66d8f08342 100644--- a/datafusion-examples/examples/custom_data_source/main.rs+++ b/datafusion-examples/examples/custom_data_source/main.rs@@ -21,7 +21,7 @@//!//! ## Usage//! ```bash-//! cargo run --example custom_data_source -- [csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|file_stream_provider]+//! cargo run --example custom_data_source -- [csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]//! ```//!//! Each subcommand runs a corresponding example:@@ -30,6 +30,7 @@//! - `custom_datasource` — run queries against a custom datasource (TableProvider)//! - `custom_file_casts` — implement custom casting rules to adapt file schemas//! - `custom_file_format` — write data to a custom file format+//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter//! - `file_stream_provider` — run a query on FileStreamProvider which implements StreamProvider for reading and writing to arbitrary stream sources/sinksmod csv_json_opener;@@ -37,6 +38,7 @@ mod csv_sql_streaming;mod custom_datasource;mod custom_file_casts;mod custom_file_format;+mod default_column_values;mod file_stream_provider;use std::str::FromStr;@@ -49,6 +51,7 @@ enum ExampleKind {CustomDatasource,CustomFileCasts,CustomFileFormat,+ DefaultColumnValues,FileFtreamProvider,}@@ -60,6 +63,7 @@ impl AsRef<str> for ExampleKind {Self::CustomDatasource => "custom_datasource",Self::CustomFileCasts => "custom_file_casts",Self::CustomFileFormat => "custom_file_format",+ Self::DefaultColumnValues => "default_column_values",Self::FileFtreamProvider => "file_stream_provider",}}@@ -75,6 +79,7 @@ impl FromStr for ExampleKind {"custom_datasource" => Ok(Self::CustomDatasource),"custom_file_casts" => Ok(Self::CustomFileCasts),"custom_file_format" => Ok(Self::CustomFileFormat),+ "default_column_values" => Ok(Self::DefaultColumnValues),"file_stream_provider" => Ok(Self::FileFtreamProvider),_ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),}@@ -82,12 +87,13 @@ impl FromStr for ExampleKind {}impl ExampleKind {- const ALL: [Self; 6] = [+ const ALL: [Self; 7] = [Self::CsvJsonOpener,Self::CsvSqlStreaming,Self::CustomDatasource,Self::CustomFileCasts,Self::CustomFileFormat,+ Self::DefaultColumnValues,Self::FileFtreamProvider,];@@ -117,6 +123,9 @@ async fn main() -> Result<()> {ExampleKind::CustomDatasource => custom_datasource::custom_datasource().await?,ExampleKind::CustomFileCasts => custom_file_casts::custom_file_casts().await?,ExampleKind::CustomFileFormat => custom_file_format::custom_file_format().await?,+ ExampleKind::DefaultColumnValues => {+ default_column_values::default_column_values().await?+ }ExampleKind::FileFtreamProvider => {file_stream_provider::file_stream_provider().await?}diff --git a/datafusion-examples/examples/dataframe.rs b/datafusion-examples/examples/dataframe/dataframe.rssimilarity index 97%rename from datafusion-examples/examples/dataframe.rsrename to datafusion-examples/examples/dataframe/dataframe.rsindex a5ee571a14..7ba4464b53 100644--- a/datafusion-examples/examples/dataframe.rs+++ b/datafusion-examples/examples/dataframe/dataframe.rs@@ -15,12 +15,13 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, StringViewArray};use datafusion::arrow::datatypes::{DataType, Field, Schema};use datafusion::catalog::MemTable;use datafusion::common::config::CsvOptions;use datafusion::common::parsers::CompressionTypeVariant;-use datafusion::common::DataFusionError;use datafusion::common::ScalarValue;use datafusion::dataframe::DataFrameWriteOptions;use datafusion::error::Result;@@ -39,6 +40,7 @@ use tempfile::tempdir;/// * [read_parquet]: execute queries against parquet files/// * [read_csv]: execute queries against csv files/// * [read_memory]: execute queries against in-memory arrow data+/// * [read_memory_macro]: execute queries against in-memory arrow data using macro////// # Writing out to local storage///@@ -53,12 +55,7 @@ use tempfile::tempdir;/// * [where_scalar_subquery]: execute a scalar subquery/// * [where_in_subquery]: execute a subquery with an IN clause/// * [where_exist_subquery]: execute a subquery with an EXISTS clause-///-/// # Querying data-///-/// * [query_to_date]: execute queries against parquet files-#[tokio::main]-async fn main() -> Result<()> {+pub async fn dataframe_example() -> Result<()> {env_logger::init();// The SessionContext is the main high level API for interacting with DataFusionlet ctx = SessionContext::new();@@ -199,7 +196,7 @@ async fn read_memory_macro() -> Result<()> {/// 2. Write out a DataFrame to a parquet file/// 3. Write out a DataFrame to a csv file/// 4. Write out a DataFrame to a json file-async fn write_out(ctx: &SessionContext) -> std::result::Result<(), DataFusionError> {+async fn write_out(ctx: &SessionContext) -> Result<()> {let array = StringViewArray::from(vec!["a", "b", "c"]);let schema = Arc::new(Schema::new(vec![Field::new("tablecol1",diff --git a/datafusion-examples/examples/deserialize_to_struct.rs b/datafusion-examples/examples/dataframe/deserialize_to_struct.rssimilarity index 98%rename from datafusion-examples/examples/deserialize_to_struct.rsrename to datafusion-examples/examples/dataframe/deserialize_to_struct.rsindex d6655b3b65..e19d455541 100644--- a/datafusion-examples/examples/deserialize_to_struct.rs+++ b/datafusion-examples/examples/dataframe/deserialize_to_struct.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+//! See `main.rs` for how to run it.+use arrow::array::{AsArray, PrimitiveArray};use arrow::datatypes::{Float64Type, Int32Type};use datafusion::common::assert_batches_eq;@@ -29,8 +31,7 @@ use futures::StreamExt;/// as [ArrayRef]////// [ArrayRef]: arrow::array::ArrayRef-#[tokio::main]-async fn main() -> Result<()> {+pub async fn deserialize_to_struct() -> Result<()> {// Run a query that returns two columns of datalet ctx = SessionContext::new();let testdata = datafusion::test_util::parquet_test_data();diff --git a/datafusion-examples/examples/dataframe/main.rs b/datafusion-examples/examples/dataframe/main.rsnew file mode 100644index 0000000000..0846bce2e8--- /dev/null+++ b/datafusion-examples/examples/dataframe/main.rs@@ -0,0 +1,95 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! # These are core DataFrame API usage+//!+//! These examples demonstrate core DataFrame API usage.+//!+//! ## Usage+//! ```bash+//! cargo run --example dataframe -- [dataframe|deserialize_to_struct]+//! ```+//!+//! Each subcommand runs a corresponding example:+//! - `dataframe` — run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries+//! - `deserialize_to_struct` — convert query results (Arrow ArrayRefs) into Rust structs++mod dataframe;+mod deserialize_to_struct;++use std::str::FromStr;++use datafusion::error::{DataFusionError, Result};++enum ExampleKind {+ Dataframe,+ DeserializeToStruct,+}++impl AsRef<str> for ExampleKind {+ fn as_ref(&self) -> &str {+ match self {+ Self::Dataframe => "dataframe",+ Self::DeserializeToStruct => "deserialize_to_struct",+ }+ }+}++impl FromStr for ExampleKind {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ "dataframe" => Ok(Self::Dataframe),+ "deserialize_to_struct" => Ok(Self::DeserializeToStruct),+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),+ }+ }+}++impl ExampleKind {+ const ALL: [Self; 2] = [Self::Dataframe, Self::DeserializeToStruct];++ const EXAMPLE_NAME: &str = "dataframe";++ fn variants() -> Vec<&'static str> {+ Self::ALL.iter().map(|x| x.as_ref()).collect()+ }+}++#[tokio::main]+async fn main() -> Result<()> {+ let usage = format!(+ "Usage: cargo run --example {} -- [{}]",+ ExampleKind::EXAMPLE_NAME,+ ExampleKind::variants().join("|")+ );++ let arg = std::env::args().nth(1).ok_or_else(|| {+ eprintln!("{usage}");+ DataFusionError::Execution("Missing argument".to_string())+ })?;++ match arg.parse::<ExampleKind>()? {+ ExampleKind::Dataframe => dataframe::dataframe_example().await?,+ ExampleKind::DeserializeToStruct => {+ deserialize_to_struct::deserialize_to_struct().await?+ }+ }++ Ok(())+}