Apache DataFusion SQL Query Engine
Consolidate builtin functions examples (#18142) (#18523)
Sergey Zhukov3de195a6d4b4
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex f87f62e170..1befba6be6 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -61,12 +61,13 @@ cargo run --example dataframe- [`custom_file_format.rs`](examples/custom_file_format.rs): Write data to a custom file format- [`dataframe-to-s3.rs`](examples/external_dependency/dataframe-to-s3.rs): Run a query using a DataFrame against a parquet file from s3 and writing back to s3- [`dataframe.rs`](examples/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries. Also demonstrates the various methods to write out a DataFrame to a table, parquet file, csv file, and json file.+- [`examples/builtin_functions/date_time`](examples/builtin_functions/date_time.rs): Examples of date-time related functions and queries- [`default_column_values.rs`](examples/default_column_values.rs): Implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter- [`deserialize_to_struct.rs`](examples/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs- [`expr_api.rs`](examples/expr_api.rs): Create, execute, simplify, analyze and coerce `Expr`s- [`file_stream_provider.rs`](examples/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks.- [`flight/sql_server.rs`](examples/flight/sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from Flight and and FlightSQL (e.g. JDBC) clients-- [`function_factory.rs`](examples/function_factory.rs): Register `CREATE FUNCTION` handler to implement SQL macros+- [`examples/builtin_functions/function_factory.rs`](examples/builtin_functions/function_factory.rs): Register `CREATE FUNCTION` handler to implement SQL macros- [`memory_pool_tracking.rs`](examples/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages- [`memory_pool_execution_plan.rs`](examples/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling- [`optimizer_rule.rs`](examples/optimizer_rule.rs): Use a custom OptimizerRule to replace certain predicates@@ -81,7 +82,7 @@ cargo run --example dataframe- [`pruning.rs`](examples/pruning.rs): Use pruning to rule out files based on statistics- [`query-aws-s3.rs`](examples/external_dependency/query-aws-s3.rs): Configure `object_store` and run a query against files stored in AWS S3- [`query-http-csv.rs`](examples/query-http-csv.rs): Configure `object_store` and run a query against files vi HTTP-- [`regexp.rs`](examples/regexp.rs): Examples of using regular expression functions+- [`examples/builtin_functions/regexp.rs`](examples/builtin_functions/regexp.rs): Examples of using regular expression functions- [`remote_catalog.rs`](examples/regexp.rs): Examples of interfacing with a remote catalog (e.g. over a network)- [`examples/udf/simple_udaf.rs`](examples/udf/simple_udaf.rs): Define and invoke a User Defined Aggregate Function (UDAF)- [`examples/udf/simple_udf.rs`](examples/udf/simple_udf.rs): Define and invoke a User Defined Scalar Function (UDF)@@ -91,7 +92,6 @@ cargo run --example dataframe- [`sql_frontend.rs`](examples/sql_frontend.rs): Create LogicalPlans (only) from sql strings- [`sql_dialect.rs`](examples/sql_dialect.rs): Example of implementing a custom SQL dialect on top of `DFParser`- [`sql_query.rs`](examples/memtable.rs): Query data using SQL (in memory `RecordBatches`, local Parquet files)-- [`date_time_function.rs`](examples/date_time_function.rs): Examples of date-time related functions and queries.## Distributeddiff --git a/datafusion-examples/examples/date_time_functions.rs b/datafusion-examples/examples/builtin_functions/date_time.rssimilarity index 97%rename from datafusion-examples/examples/date_time_functions.rsrename to datafusion-examples/examples/builtin_functions/date_time.rsindex 2628319ae3..178cba979c 100644--- a/datafusion-examples/examples/date_time_functions.rs+++ b/datafusion-examples/examples/builtin_functions/date_time.rs@@ -26,8 +26,20 @@ use datafusion::common::assert_contains;use datafusion::error::Result;use datafusion::prelude::*;-#[tokio::main]-async fn main() -> Result<()> {+/// Example: Working with Date and Time Functions+///+/// This example demonstrates how to work with various date and time+/// functions in DataFusion using both the DataFrame API and SQL queries.+///+/// It includes:+/// - `make_date`: building `DATE` values from year, month, and day columns+/// - `to_date`: converting string expressions into `DATE` values+/// - `to_timestamp`: parsing strings or numeric values into `TIMESTAMP`s+/// - `to_char`: formatting dates, timestamps, and durations as strings+///+/// Together, these examples show how to create, convert, and format temporal+/// data using DataFusion’s built-in functions.+pub async fn date_time() -> Result<()> {query_make_date().await?;query_to_date().await?;query_to_timestamp().await?;diff --git a/datafusion-examples/examples/function_factory.rs b/datafusion-examples/examples/builtin_functions/function_factory.rssimilarity index 99%rename from datafusion-examples/examples/function_factory.rsrename to datafusion-examples/examples/builtin_functions/function_factory.rsindex d4312ae594..5d41e7a260 100644--- a/datafusion-examples/examples/function_factory.rs+++ b/datafusion-examples/examples/builtin_functions/function_factory.rs@@ -42,8 +42,7 @@ use std::sync::Arc;////// This example is rather simple and does not cover all cases required for a/// real implementation.-#[tokio::main]-async fn main() -> Result<()> {+pub async fn function_factory() -> Result<()> {// First we must configure the SessionContext with our function factorylet ctx = SessionContext::new()// register custom function factorydiff --git a/datafusion-examples/examples/builtin_functions/main.rs b/datafusion-examples/examples/builtin_functions/main.rsnew file mode 100644index 0000000000..3399c395bf--- /dev/null+++ b/datafusion-examples/examples/builtin_functions/main.rs@@ -0,0 +1,94 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! # These are miscellaneous function-related examples+//!+//! These examples demonstrate miscellaneous function-related features.+//!+//! Each subcommand runs a corresponding example:+//! - `date_time` — examples of date-time related functions and queries+//! - `function_factory` — register `CREATE FUNCTION` handler to implement SQL macros+//! - `regexp` — examples of using regular expression functions++mod date_time;+mod function_factory;+mod regexp;++use std::str::FromStr;++use datafusion::error::{DataFusionError, Result};++enum ExampleKind {+ DateTime,+ FunctionFactory,+ Regexp,+}++impl AsRef<str> for ExampleKind {+ fn as_ref(&self) -> &str {+ match self {+ Self::DateTime => "date_time",+ Self::FunctionFactory => "function_factory",+ Self::Regexp => "regexp",+ }+ }+}++impl FromStr for ExampleKind {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ "date_time" => Ok(Self::DateTime),+ "function_factory" => Ok(Self::FunctionFactory),+ "regexp" => Ok(Self::Regexp),+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),+ }+ }+}++impl ExampleKind {+ const ALL: [Self; 3] = [Self::DateTime, Self::FunctionFactory, Self::Regexp];++ const EXAMPLE_NAME: &str = "builtin_functions";++ fn variants() -> Vec<&'static str> {+ Self::ALL.iter().map(|x| x.as_ref()).collect()+ }+}++#[tokio::main]+async fn main() -> Result<()> {+ let usage = format!(+ "Usage: cargo run --example {} -- [{}]",+ ExampleKind::EXAMPLE_NAME,+ ExampleKind::variants().join("|")+ );++ let arg = std::env::args().nth(1).ok_or_else(|| {+ eprintln!("{usage}");+ DataFusionError::Execution("Missing argument".to_string())+ })?;++ match arg.parse::<ExampleKind>()? {+ ExampleKind::DateTime => date_time::date_time().await?,+ ExampleKind::FunctionFactory => function_factory::function_factory().await?,+ ExampleKind::Regexp => regexp::regexp().await?,+ }++ Ok(())+}diff --git a/datafusion-examples/examples/regexp.rs b/datafusion-examples/examples/builtin_functions/regexp.rssimilarity index 99%rename from datafusion-examples/examples/regexp.rsrename to datafusion-examples/examples/builtin_functions/regexp.rsindex 12d115b9b5..13c0786930 100644--- a/datafusion-examples/examples/regexp.rs+++ b/datafusion-examples/examples/builtin_functions/regexp.rs@@ -28,12 +28,11 @@ use datafusion::prelude::*;////// Supported flags can be found at/// https://docs.rs/regex/latest/regex/#grouping-and-flags-#[tokio::main]-async fn main() -> Result<()> {+pub async fn regexp() -> Result<()> {let ctx = SessionContext::new();ctx.register_csv("examples",- "../../datafusion/physical-expr/tests/data/regex.csv",+ "datafusion/physical-expr/tests/data/regex.csv",CsvReadOptions::new(),).await?;