Apache DataFusion SQL Query Engine
Automatically generate examples documentation adv (#19294) (#19750)
Sergey Zhukovbc4c245035eb
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 8dea403b73..3936253618 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -709,6 +709,11 @@ jobs:./dev/update_function_docs.shgit diff --exit-code+# This job ensures `datafusion-examples/README.md` stays in sync with the source code:+# 1. Generates README automatically using the Rust examples docs generator+# (parsing documentation from `examples/<group>/main.rs`)+# 2. Formats the generated Markdown using DataFusion's standard Prettier setup+# 3. Compares the result against the committed README.md and fails if out-of-dateexamples-docs-check:name: check example README is up-to-dateneeds: linux-build-lib@@ -721,6 +726,16 @@ jobs:with:submodules: truefetch-depth: 1++ - name: Mark repository as safe for git+ # Required for git commands inside container (avoids "dubious ownership" error)+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"++ - name: Set up Node.js (required for prettier)+ # doc_prettier_check.sh uses npx to run prettier for Markdown formatting+ uses: actions/setup-node@v4+ with:+ node-version: '18'- name: Run examples docs check scriptrun: |@@ -778,4 +793,4 @@ jobs:run: cargo msrv --output-format json --log-target stdout verify- name: Check datafusion-protoworking-directory: datafusion/proto- run: cargo msrv --output-format json --log-target stdout verify\ No newline at end of file+ run: cargo msrv --output-format json --log-target stdout verifydiff --git a/ci/scripts/check_examples_docs.sh b/ci/scripts/check_examples_docs.shindex 37b0cc088d..01cd6b930f 100755--- a/ci/scripts/check_examples_docs.sh+++ b/ci/scripts/check_examples_docs.sh@@ -17,48 +17,57 @@# specific language governing permissions and limitations# under the License.-set -euo pipefail--EXAMPLES_DIR="datafusion-examples/examples"-README="datafusion-examples/README.md"--# ffi examples are skipped because they were not part of the recent example-# consolidation work and do not follow the new grouping and execution pattern.-# They are not documented in the README using the new structure, so including-# them here would cause false CI failures.-SKIP_LIST=("ffi")--missing=0+# Generates documentation for DataFusion examples using the Rust-based+# documentation generator and verifies that the committed README.md+# is up to date.+#+# The README is generated from documentation comments in:+# datafusion-examples/examples/<group>/main.rs+#+# This script is intended to be run in CI to ensure that example+# documentation stays in sync with the code.+#+# To update the README locally, run this script and replace README.md+# with the generated output.-skip() {- local value="$1"- for item in "${SKIP_LIST[@]}"; do- if [[ "$item" == "$value" ]]; then- return 0- fi- done- return 1-}+set -euo pipefail-# collect folder names-folders=$(find "$EXAMPLES_DIR" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)+ROOT_DIR="$(git rev-parse --show-toplevel)"+EXAMPLES_DIR="$ROOT_DIR/datafusion-examples"+README="$EXAMPLES_DIR/README.md"+README_NEW="$EXAMPLES_DIR/README-NEW.md"-# collect group names from README headers-groups=$(grep "^### Group:" "$README" | sed -E 's/^### Group: `([^`]+)`.*/\1/')+echo "▶ Generating examples README (Rust generator)…"+cargo run --quiet \+ --manifest-path "$EXAMPLES_DIR/Cargo.toml" \+ --bin examples-docs \+ > "$README_NEW"-for folder in $folders; do- if skip "$folder"; then- echo "Skipped group: $folder"- continue- fi+echo "▶ Formatting generated README with Prettier…"+npx [email protected] \+ --parser markdown \+ --write "$README_NEW"- if ! echo "$groups" | grep -qx "$folder"; then- echo "Missing README entry for example group: $folder"- missing=1- fi-done+echo "▶ Comparing generated README with committed version…"-if [[ $missing -eq 1 ]]; then- echo "README is out of sync with examples"- exit 1+if ! diff -u "$README" "$README_NEW" > /tmp/examples-readme.diff; then+ echo ""+ echo "❌ Examples README is out of date."+ echo ""+ echo "The examples documentation is generated automatically from:"+ echo " - datafusion-examples/examples/<group>/main.rs"+ echo ""+ echo "To update the README locally, run:"+ echo ""+ echo " cargo run --bin examples-docs \\"+ echo " | npx [email protected] --parser markdown --write \\"+ echo " > datafusion-examples/README.md"+ echo ""+ echo "Diff:"+ echo "------------------------------------------------------------"+ cat /tmp/examples-readme.diff+ echo "------------------------------------------------------------"+ exit 1fi++echo "✅ Examples README is up-to-date."diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.tomlindex 4cbec17272..d07eb731e5 100644--- a/datafusion-examples/Cargo.toml+++ b/datafusion-examples/Cargo.toml@@ -41,7 +41,7 @@ arrow-schema = { workspace = true }datafusion = { workspace = true, default-features = true, features = ["parquet_encryption"] }datafusion-common = { workspace = true }tempfile = { workspace = true }-tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }+tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }[dev-dependencies]arrow-flight = { workspace = true }diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 61afbf6682..2cf0ec5240 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -71,15 +71,16 @@ cargo run --example dataframe -- dataframe#### Category: Single Process-| Subcommand | File Path | Description |-| --------------------- | ----------------------------------------------------------------------------------------------------- | --------------------------------------------- |-| csv_sql_streaming | [`custom_data_source/csv_sql_streaming.rs`](examples/custom_data_source/csv_sql_streaming.rs) | Run a streaming SQL query against CSV data |-| csv_json_opener | [`custom_data_source/csv_json_opener.rs`](examples/custom_data_source/csv_json_opener.rs) | Use low-level FileOpener APIs for CSV/JSON |-| custom_datasource | [`custom_data_source/custom_datasource.rs`](examples/custom_data_source/custom_datasource.rs) | Query a custom TableProvider |-| custom_file_casts | [`custom_data_source/custom_file_casts.rs`](examples/custom_data_source/custom_file_casts.rs) | Implement custom casting rules |-| custom_file_format | [`custom_data_source/custom_file_format.rs`](examples/custom_data_source/custom_file_format.rs) | Write to a custom file format |-| default_column_values | [`custom_data_source/default_column_values.rs`](examples/custom_data_source/default_column_values.rs) | Custom default values using metadata |-| file_stream_provider | [`custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs) | Read/write via FileStreamProvider for streams |+| Subcommand | File Path | Description |+| --------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- |+| adapter_serialization | [`custom_data_source/adapter_serialization.rs`](examples/custom_data_source/adapter_serialization.rs) | Preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception |+| csv_json_opener | [`custom_data_source/csv_json_opener.rs`](examples/custom_data_source/csv_json_opener.rs) | Use low-level FileOpener APIs for CSV/JSON |+| csv_sql_streaming | [`custom_data_source/csv_sql_streaming.rs`](examples/custom_data_source/csv_sql_streaming.rs) | Run a streaming SQL query against CSV data |+| custom_datasource | [`custom_data_source/custom_datasource.rs`](examples/custom_data_source/custom_datasource.rs) | Query a custom TableProvider |+| custom_file_casts | [`custom_data_source/custom_file_casts.rs`](examples/custom_data_source/custom_file_casts.rs) | Implement custom casting rules |+| custom_file_format | [`custom_data_source/custom_file_format.rs`](examples/custom_data_source/custom_file_format.rs) | Write to a custom file format |+| default_column_values | [`custom_data_source/default_column_values.rs`](examples/custom_data_source/default_column_values.rs) | Custom default values using metadata |+| file_stream_provider | [`custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs) | Read/write via FileStreamProvider for streams |## Data IO Examples@@ -143,8 +144,8 @@ cargo run --example dataframe -- dataframe| Subcommand | File Path | Description || ---------- | ------------------------------------------------------- | ------------------------------------------------------ |-| server | [`flight/server.rs`](examples/flight/server.rs) | Run DataFusion server accepting FlightSQL/JDBC queries || client | [`flight/client.rs`](examples/flight/client.rs) | Execute SQL queries via Arrow Flight protocol |+| server | [`flight/server.rs`](examples/flight/server.rs) | Run DataFusion server accepting FlightSQL/JDBC queries || sql_server | [`flight/sql_server.rs`](examples/flight/sql_server.rs) | Standalone SQL server for JDBC clients |## Proto Examples@@ -153,9 +154,10 @@ cargo run --example dataframe -- dataframe#### Category: Single Process-| Subcommand | File Path | Description |-| ------------------------ | --------------------------------------------------------------------------------- | --------------------------------------------------------------- |-| composed_extension_codec | [`proto/composed_extension_codec.rs`](examples/proto/composed_extension_codec.rs) | Use multiple extension codecs for serialization/deserialization |+| Subcommand | File Path | Description |+| ------------------------ | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- |+| composed_extension_codec | [`proto/composed_extension_codec.rs`](examples/proto/composed_extension_codec.rs) | Use multiple extension codecs for serialization/deserialization |+| expression_deduplication | [`proto/expression_deduplication.rs`](examples/proto/expression_deduplication.rs) | Example of expression caching/deduplication using the codec decorator pattern |## Query Planning Examplesdiff --git a/datafusion-examples/examples/builtin_functions/main.rs b/datafusion-examples/examples/builtin_functions/main.rsindex 638f56dfbe..42ca15f919 100644--- a/datafusion-examples/examples/builtin_functions/main.rs+++ b/datafusion-examples/examples/builtin_functions/main.rs@@ -26,9 +26,15 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `date_time` — examples of date-time related functions and queries-//! - `function_factory` — register `CREATE FUNCTION` handler to implement SQL macros-//! - `regexp` — examples of using regular expression functions+//!+//! - `date_time`+//! (file: date_time.rs, desc: Examples of date-time related functions and queries)+//!+//! - `function_factory`+//! (file: function_factory.rs, desc: Register `CREATE FUNCTION` handler to implement SQL macros)+//!+//! - `regexp`+//! (file: regexp.rs, desc: Examples of using regular expression functions)mod date_time;mod function_factory;diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rsindex 73e65182a0..0d21a62591 100644--- a/datafusion-examples/examples/custom_data_source/main.rs+++ b/datafusion-examples/examples/custom_data_source/main.rs@@ -26,14 +26,30 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `adapter_serialization` — preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception-//! - `csv_json_opener` — use low level FileOpener APIs to read CSV/JSON into Arrow RecordBatches-//! - `csv_sql_streaming` — build and run a streaming query plan from a SQL statement against a local CSV file-//! - `custom_datasource` — run queries against a custom datasource (TableProvider)-//! - `custom_file_casts` — implement custom casting rules to adapt file schemas-//! - `custom_file_format` — write data to a custom file format-//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter-//! - `file_stream_provider` — run a query on FileStreamProvider which implements StreamProvider for reading and writing to arbitrary stream sources/sinks+//!+//! - `adapter_serialization`+//! (file: adapter_serialization.rs, desc: Preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception)+//!+//! - `csv_json_opener`+//! (file: csv_json_opener.rs, desc: Use low-level FileOpener APIs for CSV/JSON)+//!+//! - `csv_sql_streaming`+//! (file: csv_sql_streaming.rs, desc: Run a streaming SQL query against CSV data)+//!+//! - `custom_datasource`+//! (file: custom_datasource.rs, desc: Query a custom TableProvider)+//!+//! - `custom_file_casts`+//! (file: custom_file_casts.rs, desc: Implement custom casting rules)+//!+//! - `custom_file_format`+//! (file: custom_file_format.rs, desc: Write to a custom file format)+//!+//! - `default_column_values`+//! (file: default_column_values.rs, desc: Custom default values using metadata)+//!+//! - `file_stream_provider`+//! (file: file_stream_provider.rs, desc: Read/write via FileStreamProvider for streams)mod adapter_serialization;mod csv_json_opener;diff --git a/datafusion-examples/examples/data_io/main.rs b/datafusion-examples/examples/data_io/main.rsindex 0b2bd03f7e..0039585d15 100644--- a/datafusion-examples/examples/data_io/main.rs+++ b/datafusion-examples/examples/data_io/main.rs@@ -26,16 +26,36 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `catalog` — register the table into a custom catalog-//! - `json_shredding` — shows how to implement custom filter rewriting for JSON shredding-//! - `parquet_adv_idx` — create a detailed secondary index that covers the contents of several parquet files-//! - `parquet_emb_idx` — store a custom index inside a Parquet file and use it to speed up queries-//! - `parquet_enc_with_kms` — read and write encrypted Parquet files using an encryption factory-//! - `parquet_enc` — read and write encrypted Parquet files using DataFusion-//! - `parquet_exec_visitor` — extract statistics by visiting an ExecutionPlan after execution-//! - `parquet_idx` — create an secondary index over several parquet files and use it to speed up queries-//! - `query_http_csv` — configure `object_store` and run a query against files via HTTP-//! - `remote_catalog` — interfacing with a remote catalog (e.g. over a network)+//!+//! - `catalog`+//! (file: catalog.rs, desc: Register tables into a custom catalog)+//!+//! - `json_shredding`+//! (file: json_shredding.rs, desc: Implement filter rewriting for JSON shredding)+//!+//! - `parquet_adv_idx`+//! (file: parquet_advanced_index.rs, desc: Create a secondary index across multiple parquet files)+//!+//! - `parquet_emb_idx`+//! (file: parquet_embedded_index.rs, desc: Store a custom index inside Parquet files)+//!+//! - `parquet_enc`+//! (file: parquet_encrypted.rs, desc: Read & write encrypted Parquet files)+//!+//! - `parquet_enc_with_kms`+//! (file: parquet_encrypted_with_kms.rs, desc: Encrypted Parquet I/O using a KMS-backed factory)+//!+//! - `parquet_exec_visitor`+//! (file: parquet_exec_visitor.rs, desc: Extract statistics by visiting an ExecutionPlan)+//!+//! - `parquet_idx`+//! (file: parquet_index.rs, desc: Create a secondary index)+//!+//! - `query_http_csv`+//! (file: query_http_csv.rs, desc: Query CSV files via HTTP)+//!+//! - `remote_catalog`+//! (file: remote_catalog.rs, desc: Interact with a remote catalog)mod catalog;mod json_shredding;diff --git a/datafusion-examples/examples/dataframe/main.rs b/datafusion-examples/examples/dataframe/main.rsindex 8c294e2f4e..25b5377d38 100644--- a/datafusion-examples/examples/dataframe/main.rs+++ b/datafusion-examples/examples/dataframe/main.rs@@ -26,8 +26,15 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `dataframe` — run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries-//! - `deserialize_to_struct` — convert query results (Arrow ArrayRefs) into Rust structs+//!+//! - `cache_factory`+//! (file: cache_factory.rs, desc: Custom lazy caching for DataFrames using `CacheFactory`)+//+//! - `dataframe`+//! (file: dataframe.rs, desc: Query DataFrames from various sources and write output)+//!+//! - `deserialize_to_struct`+//! (file: deserialize_to_struct.rs, desc: Convert Arrow arrays into Rust structs)mod cache_factory;mod dataframe;diff --git a/datafusion-examples/examples/execution_monitoring/main.rs b/datafusion-examples/examples/execution_monitoring/main.rsindex 07de57f6b8..8f80c36929 100644--- a/datafusion-examples/examples/execution_monitoring/main.rs+++ b/datafusion-examples/examples/execution_monitoring/main.rs@@ -26,9 +26,15 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `mem_pool_exec_plan` — shows how to implement memory-aware ExecutionPlan with memory reservation and spilling-//! - `mem_pool_tracking` — demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages-//! - `tracing` — demonstrates the tracing injection feature for the DataFusion runtime+//!+//! - `mem_pool_exec_plan`+//! (file: memory_pool_execution_plan.rs, desc: Memory-aware ExecutionPlan with spilling)+//!+//! - `mem_pool_tracking`+//! (file: memory_pool_tracking.rs, desc: Demonstrates memory tracking)+//!+//! - `tracing`+//! (file: tracing.rs, desc: Demonstrates tracing integration)mod memory_pool_execution_plan;mod memory_pool_tracking;diff --git a/datafusion-examples/examples/external_dependency/main.rs b/datafusion-examples/examples/external_dependency/main.rsindex 0a9a2cd237..447e7d38bd 100644--- a/datafusion-examples/examples/external_dependency/main.rs+++ b/datafusion-examples/examples/external_dependency/main.rs@@ -26,8 +26,12 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `dataframe_to_s3` — run a query using a DataFrame against a parquet file from AWS S3 and writing back to AWS S3-//! - `query_aws_s3` — configure `object_store` and run a query against files stored in AWS S3+//!+//! - `dataframe_to_s3`+//! (file: dataframe_to_s3.rs, desc: Query DataFrames and write results to S3)+//!+//! - `query_aws_s3`+//! (file: query_aws_s3.rs, desc: Query S3-backed data using object_store)mod dataframe_to_s3;mod query_aws_s3;diff --git a/datafusion-examples/examples/flight/main.rs b/datafusion-examples/examples/flight/main.rsindex 6f20f576d3..426e806486 100644--- a/datafusion-examples/examples/flight/main.rs+++ b/datafusion-examples/examples/flight/main.rs@@ -29,9 +29,15 @@//! Note: The Flight server must be started in a separate process//! before running the `client` example. Therefore, running `all` will//! not produce a full server+client workflow automatically.-//! - `client` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol-//! - `server` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol-//! - `sql_server` — run DataFusion as a standalone process and execute SQL queries from JDBC clients+//!+//! - `client`+//! (file: client.rs, desc: Execute SQL queries via Arrow Flight protocol)+//!+//! - `server`+//! (file: server.rs, desc: Run DataFusion server accepting FlightSQL/JDBC queries)+//!+//! - `sql_server`+//! (file: sql_server.rs, desc: Standalone SQL server for JDBC clients)mod client;mod server;diff --git a/datafusion-examples/examples/proto/main.rs b/datafusion-examples/examples/proto/main.rsindex 16fcd70f34..3f525b5d46 100644--- a/datafusion-examples/examples/proto/main.rs+++ b/datafusion-examples/examples/proto/main.rs@@ -21,13 +21,17 @@//!//! ## Usage//! ```bash-//! cargo run --example proto -- [all|composed_extension_codec]+//! cargo run --example proto -- [all|composed_extension_codec|expression_deduplication]//! ```//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `composed_extension_codec` — example of using multiple extension codecs for serialization / deserialization-//! - `expression_deduplication` — example of expression caching/deduplication using the codec decorator pattern+//!+//! - `composed_extension_codec`+//! (file: composed_extension_codec.rs, desc: Use multiple extension codecs for serialization/deserialization)+//!+//! - `expression_deduplication`+//! (file: expression_deduplication.rs, desc: Example of expression caching/deduplication using the codec decorator pattern)mod composed_extension_codec;mod expression_deduplication;diff --git a/datafusion-examples/examples/query_planning/main.rs b/datafusion-examples/examples/query_planning/main.rsindex ec21c3ea5a..d3f99aedce 100644--- a/datafusion-examples/examples/query_planning/main.rs+++ b/datafusion-examples/examples/query_planning/main.rs@@ -26,14 +26,30 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `analyzer_rule` — use a custom AnalyzerRule to change a query's semantics (row level access control)-//! - `expr_api` — create, execute, simplify, analyze and coerce `Expr`s-//! - `optimizer_rule` — use a custom OptimizerRule to replace certain predicates-//! - `parse_sql_expr` — parse SQL text into DataFusion `Expr`-//! - `plan_to_sql` — generate SQL from DataFusion `Expr` and `LogicalPlan`-//! - `planner_api` — APIs to manipulate logical and physical plans-//! - `pruning` — APIs to manipulate logical and physical plans-//! - `thread_pools` — demonstrate TrackConsumersPool for memory tracking and debugging with enhanced error messages and shows how to implement memory-aware ExecutionPlan with memory reservation and spilling+//!+//! - `analyzer_rule`+//! (file: analyzer_rule.rs, desc: Custom AnalyzerRule to change query semantics)+//!+//! - `expr_api`+//! (file: expr_api.rs, desc: Create, execute, analyze, and coerce Exprs)+//!+//! - `optimizer_rule`+//! (file: optimizer_rule.rs, desc: Replace predicates via a custom OptimizerRule)+//!+//! - `parse_sql_expr`+//! (file: parse_sql_expr.rs, desc: Parse SQL into DataFusion Expr)+//!+//! - `plan_to_sql`+//! (file: plan_to_sql.rs, desc: Generate SQL from expressions or plans)+//!+//! - `planner_api`+//! (file: planner_api.rs, desc: APIs for logical and physical plan manipulation)+//!+//! - `pruning`+//! (file: pruning.rs, desc: Use pruning to skip irrelevant files)+//!+//! - `thread_pools`+//! (file: thread_pools.rs, desc: Configure custom thread pools for DataFusion execution)mod analyzer_rule;mod expr_api;diff --git a/datafusion-examples/examples/relation_planner/main.rs b/datafusion-examples/examples/relation_planner/main.rsindex 15079f6446..babc0d3714 100644--- a/datafusion-examples/examples/relation_planner/main.rs+++ b/datafusion-examples/examples/relation_planner/main.rs@@ -27,9 +27,15 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `match_recognize` — MATCH_RECOGNIZE pattern matching on event streams-//! - `pivot_unpivot` — PIVOT and UNPIVOT operations for reshaping data-//! - `table_sample` — TABLESAMPLE clause for sampling rows from tables+//!+//! - `match_recognize`+//! (file: match_recognize.rs, desc: Implement MATCH_RECOGNIZE pattern matching)+//!+//! - `pivot_unpivot`+//! (file: pivot_unpivot.rs, desc: Implement PIVOT / UNPIVOT)+//!+//! - `table_sample`+//! (file: table_sample.rs, desc: Implement TABLESAMPLE)//!//! ## Snapshot Testing//!diff --git a/datafusion-examples/examples/sql_ops/main.rs b/datafusion-examples/examples/sql_ops/main.rsindex aaab7778be..ce7be8fa2b 100644--- a/datafusion-examples/examples/sql_ops/main.rs+++ b/datafusion-examples/examples/sql_ops/main.rs@@ -26,10 +26,18 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `analysis` — analyse SQL queries with DataFusion structures-//! - `custom_sql_parser` — implementing a custom SQL parser to extend DataFusion-//! - `frontend` — create LogicalPlans (only) from sql strings-//! - `query` — query data using SQL (in memory RecordBatches, local Parquet files)+//!+//! - `analysis`+//! (file: analysis.rs, desc: Analyze SQL queries)+//!+//! - `custom_sql_parser`+//! (file: custom_sql_parser.rs, desc: Implement a custom SQL parser to extend DataFusion)+//!+//! - `frontend`+//! (file: frontend.rs, desc: Build LogicalPlans from SQL)+//!+//! - `query`+//! (file: query.rs, desc: Query data using SQL)mod analysis;mod custom_sql_parser;diff --git a/datafusion-examples/examples/udf/main.rs b/datafusion-examples/examples/udf/main.rsindex aff20e7754..e024e466ab 100644--- a/datafusion-examples/examples/udf/main.rs+++ b/datafusion-examples/examples/udf/main.rs@@ -26,14 +26,30 @@//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module-//! - `adv_udaf` — user defined aggregate function example-//! - `adv_udf` — user defined scalar function example-//! - `adv_udwf` — user defined window function example-//! - `async_udf` — asynchronous user defined function example-//! - `udaf` — simple user defined aggregate function example-//! - `udf` — simple user defined scalar function example-//! - `udtf` — simple user defined table function example-//! - `udwf` — simple user defined window function example+//!+//! - `adv_udaf`+//! (file: advanced_udaf.rs, desc: Advanced User Defined Aggregate Function (UDAF))+//!+//! - `adv_udf`+//! (file: advanced_udf.rs, desc: Advanced User Defined Scalar Function (UDF))+//!+//! - `adv_udwf`+//! (file: advanced_udwf.rs, desc: Advanced User Defined Window Function (UDWF))+//!+//! - `async_udf`+//! (file: async_udf.rs, desc: Asynchronous User Defined Scalar Function)+//!+//! - `udaf`+//! (file: simple_udaf.rs, desc: Simple UDAF example)+//!+//! - `udf`+//! (file: simple_udf.rs, desc: Simple UDF example)+//!+//! - `udtf`+//! (file: simple_udtf.rs, desc: Simple UDTF example)+//!+//! - `udwf`+//! (file: simple_udwf.rs, desc: Simple UDWF example)mod advanced_udaf;mod advanced_udf;diff --git a/datafusion-examples/src/bin/examples-docs.rs b/datafusion-examples/src/bin/examples-docs.rsnew file mode 100644index 0000000000..09a07e5fcd--- /dev/null+++ b/datafusion-examples/src/bin/examples-docs.rs@@ -0,0 +1,45 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Generates Markdown documentation for DataFusion example groups.+//!+//! This binary scans `datafusion-examples/examples`, extracts structured+//! documentation from each group's `main.rs` file, and renders a README-style+//! Markdown document.+//!+//! By default, documentation is generated for all example groups. If a group+//! name is provided as the first CLI argument, only that group is rendered.+//!+//! ## Usage+//!+//! ```bash+//! # Generate docs for all example groups+//! cargo run --bin examples-docs+//!+//! # Generate docs for a single group+//! cargo run --bin examples-docs -- dataframe+//! ```++use datafusion_examples::utils::examples_docs::{RepoLayout, generate_examples_readme};++fn main() -> Result<(), Box<dyn std::error::Error>> {+ let layout = RepoLayout::detect()?;+ let group = std::env::args().nth(1);+ let markdown = generate_examples_readme(&layout, group.as_deref())?;+ print!("{markdown}");+ Ok(())+}diff --git a/datafusion-examples/src/utils/examples_docs.rs b/datafusion-examples/src/utils/examples_docs.rsnew file mode 100644index 0000000000..0efa4928c0--- /dev/null+++ b/datafusion-examples/src/utils/examples_docs.rs@@ -0,0 +1,684 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Documentation generator for DataFusion examples.+//!+//! # Design goals+//!+//! - Keep README.md in sync with runnable examples+//! - Fail fast on malformed documentation+//!+//! # Overview+//!+//! Each example group corresponds to a directory under+//! `datafusion-examples/examples/<group>` containing a `main.rs` file.+//! Documentation is extracted from structured `//!` comments in that file.+//!+//! For each example group, the generator produces:+//!+//! ```text+//! ## <Group Name> Examples+//! ### Group: `<group>`+//! #### Category: Single Process | Distributed+//!+//! | Subcommand | File Path | Description |+//! ```+//!+//! # Usage+//!+//! Generate documentation for a single group only:+//!+//! ```bash+//! cargo run --bin examples-docs -- dataframe+//! ```+//!+//! Generate documentation for all examples:+//!+//! ```bash+//! cargo run --bin examples-docs+//! ```++use std::fs;+use std::path::{Path, PathBuf};++use datafusion::error::{DataFusionError, Result};++const STATIC_HEADER: &str = r#"<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# DataFusion Examples++This crate includes end to end, highly commented examples of how to use+various DataFusion APIs to help you get started.++## Prerequisites++Run `git submodule update --init` to init test files.++## Running Examples++To run an example, use the `cargo run` command, such as:++```bash+git clone https://github.com/apache/datafusion+cd datafusion+# Download test data+git submodule update --init++# Change to the examples directory+cd datafusion-examples/examples++# Run all examples in a group+cargo run --example <group> -- all++# Run a specific example within a group+cargo run --example <group> -- <subcommand>++# Run all examples in the `dataframe` group+cargo run --example dataframe -- all++# Run a single example from the `dataframe` group+# (apply the same pattern for any other group)+cargo run --example dataframe -- dataframe+```+"#;++const ABBREVIATIONS: &[(&str, &str)] = &[+ ("dataframe", "DataFrame"),+ ("io", "IO"),+ ("sql", "SQL"),+ ("udf", "UDF"),+];++/// Describes the layout of a DataFusion repository.+///+/// This type centralizes knowledge about where example-related+/// directories live relative to the repository root.+#[derive(Debug, Clone)]+pub struct RepoLayout {+ root: PathBuf,+}++impl From<&Path> for RepoLayout {+ fn from(path: &Path) -> Self {+ Self {+ root: path.to_path_buf(),+ }+ }+}++impl RepoLayout {+ /// Creates a layout from an explicit repository root.+ pub fn from_root(root: PathBuf) -> Self {+ Self { root }+ }++ /// Detects the repository root based on `CARGO_MANIFEST_DIR`.+ ///+ /// This is intended for use from binaries inside the workspace.+ pub fn detect() -> Result<Self> {+ let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));++ let root = manifest_dir.parent().ok_or_else(|| {+ DataFusionError::Execution(+ "CARGO_MANIFEST_DIR does not have a parent".to_string(),+ )+ })?;++ Ok(Self {+ root: root.to_path_buf(),+ })+ }++ /// Returns the repository root directory.+ pub fn root(&self) -> &Path {+ &self.root+ }++ /// Returns the `datafusion-examples/examples` directory.+ pub fn examples_root(&self) -> PathBuf {+ self.root.join("datafusion-examples").join("examples")+ }++ /// Returns the directory for a single example group.+ ///+ /// Example: `examples/udf`+ pub fn example_group_dir(&self, group: &str) -> PathBuf {+ self.examples_root().join(group)+ }+}++/// A group of related examples (e.g. `builtin_functions`, `udf`).+///+/// Each group corresponds to a directory containing a `main.rs` file+/// with structured documentation comments.+#[derive(Debug)]+pub struct ExampleGroup {+ pub name: GroupName,+ pub examples: Vec<ExampleEntry>,+ pub category: Category,+}++impl ExampleGroup {+ /// Parses an example group from its directory.+ ///+ /// The group name is derived from the directory name, and example+ /// entries are extracted from `main.rs`.+ pub fn from_dir(dir: &Path, category: Category) -> Result<Self> {+ let raw_name = dir+ .file_name()+ .and_then(|s| s.to_str())+ .ok_or_else(|| {+ DataFusionError::Execution("Invalid example group dir".to_string())+ })?+ .to_string();++ let name = GroupName::from_dir_name(raw_name);+ let main_rs = dir.join("main.rs");+ let examples = parse_main_rs_docs(&main_rs)?;++ Ok(Self {+ name,+ examples,+ category,+ })+ }++ // Renders this example group as a Markdown section.+ pub fn render_markdown(&self) -> String {+ let mut out = String::new();+ out.push_str(&format!("\n## {} Examples\n\n", self.name.title()));+ out.push_str(&format!("### Group: `{}`\n\n", self.name.raw()));+ out.push_str(&format!("#### Category: {}\n\n", self.category.name()));+ out.push_str("| Subcommand | File Path | Description |\n");+ out.push_str("| --- | --- | --- |\n");++ for ex in &self.examples {+ out.push_str(&format!(+ "| {} | [`{}/{}`](examples/{}/{}) | {} |\n",+ ex.subcommand,+ self.name.raw(),+ ex.file,+ self.name.raw(),+ ex.file,+ ex.desc+ ));+ }++ out+ }+}++/// Represents an example group name in both raw and human-readable forms.+///+/// For example:+/// - raw: `builtin_functions`+/// - title: `Builtin Functions`+#[derive(Debug)]+pub struct GroupName {+ raw: String,+ title: String,+}++impl GroupName {+ /// Creates a group name from a directory name.+ pub fn from_dir_name(raw: String) -> Self {+ let title = raw+ .split('_')+ .map(format_part)+ .collect::<Vec<_>>()+ .join(" ");++ Self { raw, title }+ }++ /// Returns the raw group name (directory name).+ pub fn raw(&self) -> &str {+ &self.raw+ }++ /// Returns a title-cased name for documentation.+ pub fn title(&self) -> &str {+ &self.title+ }+}++/// A single runnable example within a group.+///+/// Each entry corresponds to a subcommand documented in `main.rs`.+#[derive(Debug)]+pub struct ExampleEntry {+ /// CLI subcommand name.+ pub subcommand: String,+ /// Rust source file name.+ pub file: String,+ /// Human-readable description.+ pub desc: String,+}++/// Execution category of an example group.+#[derive(Debug, Default)]+pub enum Category {+ /// Runs in a single process.+ #[default]+ SingleProcess,+ /// Requires a distributed setup.+ Distributed,+}++impl Category {+ /// Returns the display name used in documentation.+ pub fn name(&self) -> &str {+ match self {+ Self::SingleProcess => "Single Process",+ Self::Distributed => "Distributed",+ }+ }++ /// Determines the category for a group by name.+ pub fn for_group(name: &str) -> Self {+ match name {+ "flight" => Category::Distributed,+ _ => Category::SingleProcess,+ }+ }+}++/// Generates Markdown documentation for DataFusion examples.+///+/// If `group` is `None`, documentation is generated for all example groups.+/// If `group` is `Some`, only that group is rendered.+///+/// # Errors+///+/// Returns an error if:+/// - the requested group does not exist+/// - a `main.rs` file is missing+/// - documentation comments are malformed+pub fn generate_examples_readme(+ layout: &RepoLayout,+ group: Option<&str>,+) -> Result<String> {+ let examples_root = layout.examples_root();++ let mut out = String::new();+ out.push_str(STATIC_HEADER);++ let group_dirs: Vec<PathBuf> = match group {+ Some(name) => {+ let dir = examples_root.join(name);+ if !dir.is_dir() {+ return Err(DataFusionError::Execution(format!(+ "Example group `{name}` does not exist"+ )));+ }+ vec![dir]+ }+ None => discover_example_groups(&examples_root)?,+ };++ for group_dir in group_dirs {+ let raw_name =+ group_dir+ .file_name()+ .and_then(|s| s.to_str())+ .ok_or_else(|| {+ DataFusionError::Execution("Invalid example group dir".to_string())+ })?;++ let category = Category::for_group(raw_name);+ let group = ExampleGroup::from_dir(&group_dir, category)?;++ out.push_str(&group.render_markdown());+ }++ Ok(out)+}++/// Parses example entries from a group's `main.rs` file.+pub fn parse_main_rs_docs(path: &Path) -> Result<Vec<ExampleEntry>> {+ let content = fs::read_to_string(path)?;+ let mut entries = Vec::new();+ let mut pending_subcommand: Option<String> = None;++ for raw_line in content.lines() {+ let line = raw_line.trim();++ if let Some(sub) = parse_subcommand_line(line) {+ if sub != "all" {+ pending_subcommand = Some(sub);+ }+ continue;+ }++ if let Some((file, desc)) = parse_metadata_line(line) {+ let subcommand = pending_subcommand.take().ok_or_else(|| {+ DataFusionError::Execution(+ "Metadata without preceding subcommand".to_string(),+ )+ })?;++ entries.push(ExampleEntry {+ subcommand,+ file,+ desc,+ });+ }+ }++ Ok(entries)+}++/// Parses a subcommand declaration line from `main.rs` docs.+///+/// Expected format:+/// ```text+/// //! - `<subcommand>`+/// ```+fn parse_subcommand_line(line: &str) -> Option<String> {+ line.strip_prefix("//! - `")+ .and_then(|rest| rest.strip_suffix('`'))+ .map(|s| s.to_string())+}++/// Parses example metadata (file name and description) from `main.rs` docs.+///+/// Expected format:+/// ```text+/// //! (file: <file>.rs, desc: <description>)+/// ```+fn parse_metadata_line(line: &str) -> Option<(String, String)> {+ let line = line.strip_prefix("//!").map(str::trim)?;+ if !line.starts_with("(file:") || !line.ends_with(')') {+ return None;+ }+ let inner = line.strip_prefix('(')?.strip_suffix(')')?;+ let (file_part, desc_part) = inner.split_once(", desc:")?;+ let file = file_part.strip_prefix("file:")?.trim().to_string();+ let desc = desc_part.trim().to_string();+ Some((file, desc))+}++/// Discovers all example group directories under the given root.+fn discover_example_groups(root: &Path) -> Result<Vec<PathBuf>> {+ let mut groups = Vec::new();+ for entry in fs::read_dir(root)? {+ let entry = entry?;+ let path = entry.path();++ if path.is_dir() && path.join("main.rs").exists() {+ groups.push(path);+ }+ }++ groups.sort();++ Ok(groups)+}++/// Formats a single group-name segment for display.+///+/// This function applies DataFusion-specific capitalization rules:+/// - Known abbreviations (e.g. `sql`, `io`, `udf`) are rendered in all caps+/// - All other segments fall back to standard Title Case+fn format_part(part: &str) -> String {+ let lower = part.to_ascii_lowercase();++ if let Some((_, replacement)) = ABBREVIATIONS.iter().find(|(k, _)| *k == lower) {+ return replacement.to_string();+ }++ let mut chars = part.chars();+ match chars.next() {+ Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),+ None => String::new(),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ use tempfile::TempDir;++ #[test]+ fn all_subcommand_is_ignored() -> Result<()> {+ let tmp = TempDir::new().unwrap();+ let group_dir = tmp.path().join("foo");+ fs::create_dir(&group_dir)?;++ fs::write(+ group_dir.join("main.rs"),+ r#"+//! - `all`+//!+//! - `foo`+//! (file: foo.rs, desc: foo example)+"#,+ )?;++ let group = ExampleGroup::from_dir(&group_dir, Category::SingleProcess)?;++ assert_eq!(group.examples.len(), 1);+ assert_eq!(group.examples[0].subcommand, "foo");++ Ok(())+ }++ #[test]+ fn parse_subcommand_line_works() {+ let line = "//! - `date_time`";+ let sub = parse_subcommand_line(line).unwrap();+ assert_eq!(sub, "date_time");+ }++ #[test]+ fn parse_metadata_line_works() {+ let line =+ "//! (file: date_time.rs, desc: Examples of date-time related functions)";+ let (file, desc) = parse_metadata_line(line).unwrap();+ assert_eq!(file, "date_time.rs");+ assert_eq!(desc, "Examples of date-time related functions");++ let line = "//! (file: foo.rs, desc: Foo, bar, baz)";+ let (file, desc) = parse_metadata_line(line).unwrap();+ assert_eq!(file, "foo.rs");+ assert_eq!(desc, "Foo, bar, baz");+ }++ #[test]+ fn metadata_without_subcommand_fails() {+ let tmp = TempDir::new().unwrap();+ let group_dir = tmp.path().join("bad");+ fs::create_dir(&group_dir).unwrap();++ fs::write(+ group_dir.join("main.rs"),+ "//! (file: foo.rs, desc: missing subcommand)",+ )+ .unwrap();++ let err =+ ExampleGroup::from_dir(&group_dir, Category::SingleProcess).unwrap_err();++ assert!(+ err.to_string()+ .contains("Metadata without preceding subcommand"),+ "unexpected error: {err}"+ );+ }++ #[test]+ fn parse_group_example_works() -> Result<()> {+ let tmp = TempDir::new().unwrap();++ // Simulate: examples/builtin_functions/+ let group_dir = tmp.path().join("builtin_functions");+ fs::create_dir(&group_dir)?;++ // Write a fake main.rs with docs+ let main_rs = group_dir.join("main.rs");+ fs::write(+ &main_rs,+ r#"+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.+//+//! # These are miscellaneous function-related examples+//!+//! These examples demonstrate miscellaneous function-related features.+//!+//! ## Usage+//! ```bash+//! cargo run --example builtin_functions -- [all|date_time|function_factory|regexp]+//! ```+//!+//! Each subcommand runs a corresponding example:+//! - `all` — run all examples included in this module+//!+//! - `date_time`+//! (file: date_time.rs, desc: Examples of date-time related functions and queries)+//!+//! - `function_factory`+//! (file: function_factory.rs, desc: Register `CREATE FUNCTION` handler to implement SQL macros)+//!+//! - `regexp`+//! (file: regexp.rs, desc: Examples of using regular expression functions)+"#,+ )?;++ let group = ExampleGroup::from_dir(&group_dir, Category::SingleProcess)?;++ // Assert group-level data+ assert_eq!(group.name.title(), "Builtin Functions");+ assert_eq!(group.examples.len(), 3);++ // Assert 1 example+ assert_eq!(group.examples[0].subcommand, "date_time");+ assert_eq!(group.examples[0].file, "date_time.rs");+ assert_eq!(+ group.examples[0].desc,+ "Examples of date-time related functions and queries"+ );++ // Assert 2 example+ assert_eq!(group.examples[1].subcommand, "function_factory");+ assert_eq!(group.examples[1].file, "function_factory.rs");+ assert_eq!(+ group.examples[1].desc,+ "Register `CREATE FUNCTION` handler to implement SQL macros"+ );++ // Assert 3 example+ assert_eq!(group.examples[2].subcommand, "regexp");+ assert_eq!(group.examples[2].file, "regexp.rs");+ assert_eq!(+ group.examples[2].desc,+ "Examples of using regular expression functions"+ );++ Ok(())+ }++ #[test]+ fn single_group_generation_works() {+ let tmp = TempDir::new().unwrap();+ // Fake repo root+ let layout = RepoLayout::from_root(tmp.path().to_path_buf());++ // Create: datafusion-examples/examples/builtin_functions+ let examples_dir = layout.example_group_dir("builtin_functions");+ fs::create_dir_all(&examples_dir).unwrap();++ fs::write(+ examples_dir.join("main.rs"),+ "//! - `x`\n//! (file: foo.rs, desc: test)",+ )+ .unwrap();++ let out = generate_examples_readme(&layout, Some("builtin_functions")).unwrap();++ assert!(out.contains("Builtin Functions"));+ }++ #[test]+ fn single_group_generation_fails_if_group_missing() {+ let tmp = TempDir::new().unwrap();+ let layout = RepoLayout::from_root(tmp.path().to_path_buf());++ let err = generate_examples_readme(&layout, Some("missing_group")).unwrap_err();++ assert!(+ err.to_string()+ .contains("Example group `missing_group` does not exist"),+ "unexpected error: {err}"+ );+ }++ #[test]+ fn group_name_title_is_human_readable() {+ let name = GroupName::from_dir_name("very_long_group_name".to_string());+ assert_eq!(name.title(), "Very Long Group Name");+ }++ #[test]+ fn group_name_handles_abbreviations() {+ assert_eq!(+ GroupName::from_dir_name("dataframe".to_string()).title(),+ "DataFrame"+ );+ assert_eq!(+ GroupName::from_dir_name("data_io".to_string()).title(),+ "Data IO"+ );+ assert_eq!(+ GroupName::from_dir_name("sql_ops".to_string()).title(),+ "SQL Ops"+ );+ assert_eq!(GroupName::from_dir_name("udf".to_string()).title(), "UDF");+ }+}diff --git a/datafusion-examples/src/utils/mod.rs b/datafusion-examples/src/utils/mod.rsindex b9e5b487db..aa44491296 100644--- a/datafusion-examples/src/utils/mod.rs+++ b/datafusion-examples/src/utils/mod.rs@@ -17,5 +17,6 @@mod csv_to_parquet;pub mod datasets;+pub mod examples_docs;pub use csv_to_parquet::write_csv_to_parquet;