Skip to content

Apache DataFusion SQL Query Engine

Automatically generate examples documentation adv (#19294) (#19750)

Sergey Zhukovbc4c245035eb
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 8dea403b73..3936253618 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -709,6 +709,11 @@ jobs:
./dev/update_function_docs.sh
git diff --exit-code
+# This job ensures `datafusion-examples/README.md` stays in sync with the source code:
+# 1. Generates README automatically using the Rust examples docs generator
+# (parsing documentation from `examples/<group>/main.rs`)
+# 2. Formats the generated Markdown using DataFusion's standard Prettier setup
+# 3. Compares the result against the committed README.md and fails if out-of-date
examples-docs-check:
name: check example README is up-to-date
needs: linux-build-lib
@@ -721,6 +726,16 @@ jobs:
with:
submodules: true
fetch-depth: 1
+
+ - name: Mark repository as safe for git
+ # Required for git commands inside container (avoids "dubious ownership" error)
+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+ - name: Set up Node.js (required for prettier)
+ # doc_prettier_check.sh uses npx to run prettier for Markdown formatting
+ uses: actions/setup-node@v4
+ with:
+ node-version: '18'
- name: Run examples docs check script
run: |
@@ -778,4 +793,4 @@ jobs:
run: cargo msrv --output-format json --log-target stdout verify
- name: Check datafusion-proto
working-directory: datafusion/proto
- run: cargo msrv --output-format json --log-target stdout verify
\ No newline at end of file
+ run: cargo msrv --output-format json --log-target stdout verify
diff --git a/ci/scripts/check_examples_docs.sh b/ci/scripts/check_examples_docs.sh
index 37b0cc088d..01cd6b930f 100755
--- a/ci/scripts/check_examples_docs.sh
+++ b/ci/scripts/check_examples_docs.sh
@@ -17,48 +17,57 @@
# specific language governing permissions and limitations
# under the License.
-set -euo pipefail
-
-EXAMPLES_DIR="datafusion-examples/examples"
-README="datafusion-examples/README.md"
-
-# ffi examples are skipped because they were not part of the recent example
-# consolidation work and do not follow the new grouping and execution pattern.
-# They are not documented in the README using the new structure, so including
-# them here would cause false CI failures.
-SKIP_LIST=("ffi")
-
-missing=0
+# Generates documentation for DataFusion examples using the Rust-based
+# documentation generator and verifies that the committed README.md
+# is up to date.
+#
+# The README is generated from documentation comments in:
+# datafusion-examples/examples/<group>/main.rs
+#
+# This script is intended to be run in CI to ensure that example
+# documentation stays in sync with the code.
+#
+# To update the README locally, run this script and replace README.md
+# with the generated output.
-skip() {
- local value="$1"
- for item in "${SKIP_LIST[@]}"; do
- if [[ "$item" == "$value" ]]; then
- return 0
- fi
- done
- return 1
-}
+set -euo pipefail
-# collect folder names
-folders=$(find "$EXAMPLES_DIR" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
+ROOT_DIR="$(git rev-parse --show-toplevel)"
+EXAMPLES_DIR="$ROOT_DIR/datafusion-examples"
+README="$EXAMPLES_DIR/README.md"
+README_NEW="$EXAMPLES_DIR/README-NEW.md"
-# collect group names from README headers
-groups=$(grep "^### Group:" "$README" | sed -E 's/^### Group: `([^`]+)`.*/\1/')
+echo "▶ Generating examples README (Rust generator)…"
+cargo run --quiet \
+ --manifest-path "$EXAMPLES_DIR/Cargo.toml" \
+ --bin examples-docs \
+ > "$README_NEW"
-for folder in $folders; do
- if skip "$folder"; then
- echo "Skipped group: $folder"
- continue
- fi
+echo "▶ Formatting generated README with Prettier…"
+ --parser markdown \
+ --write "$README_NEW"
- if ! echo "$groups" | grep -qx "$folder"; then
- echo "Missing README entry for example group: $folder"
- missing=1
- fi
-done
+echo "▶ Comparing generated README with committed version…"
-if [[ $missing -eq 1 ]]; then
- echo "README is out of sync with examples"
- exit 1
+if ! diff -u "$README" "$README_NEW" > /tmp/examples-readme.diff; then
+ echo ""
+ echo "❌ Examples README is out of date."
+ echo ""
+ echo "The examples documentation is generated automatically from:"
+ echo " - datafusion-examples/examples/<group>/main.rs"
+ echo ""
+ echo "To update the README locally, run:"
+ echo ""
+ echo " cargo run --bin examples-docs \\"
+ echo " | npx [email protected] --parser markdown --write \\"
+ echo " > datafusion-examples/README.md"
+ echo ""
+ echo "Diff:"
+ echo "------------------------------------------------------------"
+ cat /tmp/examples-readme.diff
+ echo "------------------------------------------------------------"
+ exit 1
fi
+
+echo "✅ Examples README is up-to-date."
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 4cbec17272..d07eb731e5 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -41,7 +41,7 @@ arrow-schema = { workspace = true }
datafusion = { workspace = true, default-features = true, features = ["parquet_encryption"] }
datafusion-common = { workspace = true }
tempfile = { workspace = true }
-tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }
[dev-dependencies]
arrow-flight = { workspace = true }
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
index 61afbf6682..2cf0ec5240 100644
--- a/datafusion-examples/README.md
+++ b/datafusion-examples/README.md
@@ -71,15 +71,16 @@ cargo run --example dataframe -- dataframe
#### Category: Single Process
-| Subcommand | File Path | Description |
-| --------------------- | ----------------------------------------------------------------------------------------------------- | --------------------------------------------- |
-| csv_sql_streaming | [`custom_data_source/csv_sql_streaming.rs`](examples/custom_data_source/csv_sql_streaming.rs) | Run a streaming SQL query against CSV data |
-| csv_json_opener | [`custom_data_source/csv_json_opener.rs`](examples/custom_data_source/csv_json_opener.rs) | Use low-level FileOpener APIs for CSV/JSON |
-| custom_datasource | [`custom_data_source/custom_datasource.rs`](examples/custom_data_source/custom_datasource.rs) | Query a custom TableProvider |
-| custom_file_casts | [`custom_data_source/custom_file_casts.rs`](examples/custom_data_source/custom_file_casts.rs) | Implement custom casting rules |
-| custom_file_format | [`custom_data_source/custom_file_format.rs`](examples/custom_data_source/custom_file_format.rs) | Write to a custom file format |
-| default_column_values | [`custom_data_source/default_column_values.rs`](examples/custom_data_source/default_column_values.rs) | Custom default values using metadata |
-| file_stream_provider | [`custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs) | Read/write via FileStreamProvider for streams |
+| Subcommand | File Path | Description |
+| --------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- |
+| adapter_serialization | [`custom_data_source/adapter_serialization.rs`](examples/custom_data_source/adapter_serialization.rs) | Preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception |
+| csv_json_opener | [`custom_data_source/csv_json_opener.rs`](examples/custom_data_source/csv_json_opener.rs) | Use low-level FileOpener APIs for CSV/JSON |
+| csv_sql_streaming | [`custom_data_source/csv_sql_streaming.rs`](examples/custom_data_source/csv_sql_streaming.rs) | Run a streaming SQL query against CSV data |
+| custom_datasource | [`custom_data_source/custom_datasource.rs`](examples/custom_data_source/custom_datasource.rs) | Query a custom TableProvider |
+| custom_file_casts | [`custom_data_source/custom_file_casts.rs`](examples/custom_data_source/custom_file_casts.rs) | Implement custom casting rules |
+| custom_file_format | [`custom_data_source/custom_file_format.rs`](examples/custom_data_source/custom_file_format.rs) | Write to a custom file format |
+| default_column_values | [`custom_data_source/default_column_values.rs`](examples/custom_data_source/default_column_values.rs) | Custom default values using metadata |
+| file_stream_provider | [`custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs) | Read/write via FileStreamProvider for streams |
## Data IO Examples
@@ -143,8 +144,8 @@ cargo run --example dataframe -- dataframe
| Subcommand | File Path | Description |
| ---------- | ------------------------------------------------------- | ------------------------------------------------------ |
-| server | [`flight/server.rs`](examples/flight/server.rs) | Run DataFusion server accepting FlightSQL/JDBC queries |
| client | [`flight/client.rs`](examples/flight/client.rs) | Execute SQL queries via Arrow Flight protocol |
+| server | [`flight/server.rs`](examples/flight/server.rs) | Run DataFusion server accepting FlightSQL/JDBC queries |
| sql_server | [`flight/sql_server.rs`](examples/flight/sql_server.rs) | Standalone SQL server for JDBC clients |
## Proto Examples
@@ -153,9 +154,10 @@ cargo run --example dataframe -- dataframe
#### Category: Single Process
-| Subcommand | File Path | Description |
-| ------------------------ | --------------------------------------------------------------------------------- | --------------------------------------------------------------- |
-| composed_extension_codec | [`proto/composed_extension_codec.rs`](examples/proto/composed_extension_codec.rs) | Use multiple extension codecs for serialization/deserialization |
+| Subcommand | File Path | Description |
+| ------------------------ | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- |
+| composed_extension_codec | [`proto/composed_extension_codec.rs`](examples/proto/composed_extension_codec.rs) | Use multiple extension codecs for serialization/deserialization |
+| expression_deduplication | [`proto/expression_deduplication.rs`](examples/proto/expression_deduplication.rs) | Example of expression caching/deduplication using the codec decorator pattern |
## Query Planning Examples
diff --git a/datafusion-examples/examples/builtin_functions/main.rs b/datafusion-examples/examples/builtin_functions/main.rs
index 638f56dfbe..42ca15f919 100644
--- a/datafusion-examples/examples/builtin_functions/main.rs
+++ b/datafusion-examples/examples/builtin_functions/main.rs
@@ -26,9 +26,15 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `date_time` — examples of date-time related functions and queries
-//! - `function_factory` — register `CREATE FUNCTION` handler to implement SQL macros
-//! - `regexp` — examples of using regular expression functions
+//!
+//! - `date_time`
+//! (file: date_time.rs, desc: Examples of date-time related functions and queries)
+//!
+//! - `function_factory`
+//! (file: function_factory.rs, desc: Register `CREATE FUNCTION` handler to implement SQL macros)
+//!
+//! - `regexp`
+//! (file: regexp.rs, desc: Examples of using regular expression functions)
mod date_time;
mod function_factory;
diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rs
index 73e65182a0..0d21a62591 100644
--- a/datafusion-examples/examples/custom_data_source/main.rs
+++ b/datafusion-examples/examples/custom_data_source/main.rs
@@ -26,14 +26,30 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `adapter_serialization` — preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception
-//! - `csv_json_opener` — use low level FileOpener APIs to read CSV/JSON into Arrow RecordBatches
-//! - `csv_sql_streaming` — build and run a streaming query plan from a SQL statement against a local CSV file
-//! - `custom_datasource` — run queries against a custom datasource (TableProvider)
-//! - `custom_file_casts` — implement custom casting rules to adapt file schemas
-//! - `custom_file_format` — write data to a custom file format
-//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter
-//! - `file_stream_provider` — run a query on FileStreamProvider which implements StreamProvider for reading and writing to arbitrary stream sources/sinks
+//!
+//! - `adapter_serialization`
+//! (file: adapter_serialization.rs, desc: Preserve custom PhysicalExprAdapter information during plan serialization using PhysicalExtensionCodec interception)
+//!
+//! - `csv_json_opener`
+//! (file: csv_json_opener.rs, desc: Use low-level FileOpener APIs for CSV/JSON)
+//!
+//! - `csv_sql_streaming`
+//! (file: csv_sql_streaming.rs, desc: Run a streaming SQL query against CSV data)
+//!
+//! - `custom_datasource`
+//! (file: custom_datasource.rs, desc: Query a custom TableProvider)
+//!
+//! - `custom_file_casts`
+//! (file: custom_file_casts.rs, desc: Implement custom casting rules)
+//!
+//! - `custom_file_format`
+//! (file: custom_file_format.rs, desc: Write to a custom file format)
+//!
+//! - `default_column_values`
+//! (file: default_column_values.rs, desc: Custom default values using metadata)
+//!
+//! - `file_stream_provider`
+//! (file: file_stream_provider.rs, desc: Read/write via FileStreamProvider for streams)
mod adapter_serialization;
mod csv_json_opener;
diff --git a/datafusion-examples/examples/data_io/main.rs b/datafusion-examples/examples/data_io/main.rs
index 0b2bd03f7e..0039585d15 100644
--- a/datafusion-examples/examples/data_io/main.rs
+++ b/datafusion-examples/examples/data_io/main.rs
@@ -26,16 +26,36 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `catalog` — register the table into a custom catalog
-//! - `json_shredding` — shows how to implement custom filter rewriting for JSON shredding
-//! - `parquet_adv_idx` — create a detailed secondary index that covers the contents of several parquet files
-//! - `parquet_emb_idx` — store a custom index inside a Parquet file and use it to speed up queries
-//! - `parquet_enc_with_kms` — read and write encrypted Parquet files using an encryption factory
-//! - `parquet_enc` — read and write encrypted Parquet files using DataFusion
-//! - `parquet_exec_visitor` — extract statistics by visiting an ExecutionPlan after execution
-//! - `parquet_idx` — create an secondary index over several parquet files and use it to speed up queries
-//! - `query_http_csv` — configure `object_store` and run a query against files via HTTP
-//! - `remote_catalog` — interfacing with a remote catalog (e.g. over a network)
+//!
+//! - `catalog`
+//! (file: catalog.rs, desc: Register tables into a custom catalog)
+//!
+//! - `json_shredding`
+//! (file: json_shredding.rs, desc: Implement filter rewriting for JSON shredding)
+//!
+//! - `parquet_adv_idx`
+//! (file: parquet_advanced_index.rs, desc: Create a secondary index across multiple parquet files)
+//!
+//! - `parquet_emb_idx`
+//! (file: parquet_embedded_index.rs, desc: Store a custom index inside Parquet files)
+//!
+//! - `parquet_enc`
+//! (file: parquet_encrypted.rs, desc: Read & write encrypted Parquet files)
+//!
+//! - `parquet_enc_with_kms`
+//! (file: parquet_encrypted_with_kms.rs, desc: Encrypted Parquet I/O using a KMS-backed factory)
+//!
+//! - `parquet_exec_visitor`
+//! (file: parquet_exec_visitor.rs, desc: Extract statistics by visiting an ExecutionPlan)
+//!
+//! - `parquet_idx`
+//! (file: parquet_index.rs, desc: Create a secondary index)
+//!
+//! - `query_http_csv`
+//! (file: query_http_csv.rs, desc: Query CSV files via HTTP)
+//!
+//! - `remote_catalog`
+//! (file: remote_catalog.rs, desc: Interact with a remote catalog)
mod catalog;
mod json_shredding;
diff --git a/datafusion-examples/examples/dataframe/main.rs b/datafusion-examples/examples/dataframe/main.rs
index 8c294e2f4e..25b5377d38 100644
--- a/datafusion-examples/examples/dataframe/main.rs
+++ b/datafusion-examples/examples/dataframe/main.rs
@@ -26,8 +26,15 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `dataframe` — run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries
-//! - `deserialize_to_struct` — convert query results (Arrow ArrayRefs) into Rust structs
+//!
+//! - `cache_factory`
+//! (file: cache_factory.rs, desc: Custom lazy caching for DataFrames using `CacheFactory`)
+//
+//! - `dataframe`
+//! (file: dataframe.rs, desc: Query DataFrames from various sources and write output)
+//!
+//! - `deserialize_to_struct`
+//! (file: deserialize_to_struct.rs, desc: Convert Arrow arrays into Rust structs)
mod cache_factory;
mod dataframe;
diff --git a/datafusion-examples/examples/execution_monitoring/main.rs b/datafusion-examples/examples/execution_monitoring/main.rs
index 07de57f6b8..8f80c36929 100644
--- a/datafusion-examples/examples/execution_monitoring/main.rs
+++ b/datafusion-examples/examples/execution_monitoring/main.rs
@@ -26,9 +26,15 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `mem_pool_exec_plan` — shows how to implement memory-aware ExecutionPlan with memory reservation and spilling
-//! - `mem_pool_tracking` — demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages
-//! - `tracing` — demonstrates the tracing injection feature for the DataFusion runtime
+//!
+//! - `mem_pool_exec_plan`
+//! (file: memory_pool_execution_plan.rs, desc: Memory-aware ExecutionPlan with spilling)
+//!
+//! - `mem_pool_tracking`
+//! (file: memory_pool_tracking.rs, desc: Demonstrates memory tracking)
+//!
+//! - `tracing`
+//! (file: tracing.rs, desc: Demonstrates tracing integration)
mod memory_pool_execution_plan;
mod memory_pool_tracking;
diff --git a/datafusion-examples/examples/external_dependency/main.rs b/datafusion-examples/examples/external_dependency/main.rs
index 0a9a2cd237..447e7d38bd 100644
--- a/datafusion-examples/examples/external_dependency/main.rs
+++ b/datafusion-examples/examples/external_dependency/main.rs
@@ -26,8 +26,12 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `dataframe_to_s3` — run a query using a DataFrame against a parquet file from AWS S3 and writing back to AWS S3
-//! - `query_aws_s3` — configure `object_store` and run a query against files stored in AWS S3
+//!
+//! - `dataframe_to_s3`
+//! (file: dataframe_to_s3.rs, desc: Query DataFrames and write results to S3)
+//!
+//! - `query_aws_s3`
+//! (file: query_aws_s3.rs, desc: Query S3-backed data using object_store)
mod dataframe_to_s3;
mod query_aws_s3;
diff --git a/datafusion-examples/examples/flight/main.rs b/datafusion-examples/examples/flight/main.rs
index 6f20f576d3..426e806486 100644
--- a/datafusion-examples/examples/flight/main.rs
+++ b/datafusion-examples/examples/flight/main.rs
@@ -29,9 +29,15 @@
//! Note: The Flight server must be started in a separate process
//! before running the `client` example. Therefore, running `all` will
//! not produce a full server+client workflow automatically.
-//! - `client` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol
-//! - `server` — run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol
-//! - `sql_server` — run DataFusion as a standalone process and execute SQL queries from JDBC clients
+//!
+//! - `client`
+//! (file: client.rs, desc: Execute SQL queries via Arrow Flight protocol)
+//!
+//! - `server`
+//! (file: server.rs, desc: Run DataFusion server accepting FlightSQL/JDBC queries)
+//!
+//! - `sql_server`
+//! (file: sql_server.rs, desc: Standalone SQL server for JDBC clients)
mod client;
mod server;
diff --git a/datafusion-examples/examples/proto/main.rs b/datafusion-examples/examples/proto/main.rs
index 16fcd70f34..3f525b5d46 100644
--- a/datafusion-examples/examples/proto/main.rs
+++ b/datafusion-examples/examples/proto/main.rs
@@ -21,13 +21,17 @@
//!
//! ## Usage
//! ```bash
-//! cargo run --example proto -- [all|composed_extension_codec]
+//! cargo run --example proto -- [all|composed_extension_codec|expression_deduplication]
//! ```
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `composed_extension_codec` — example of using multiple extension codecs for serialization / deserialization
-//! - `expression_deduplication` — example of expression caching/deduplication using the codec decorator pattern
+//!
+//! - `composed_extension_codec`
+//! (file: composed_extension_codec.rs, desc: Use multiple extension codecs for serialization/deserialization)
+//!
+//! - `expression_deduplication`
+//! (file: expression_deduplication.rs, desc: Example of expression caching/deduplication using the codec decorator pattern)
mod composed_extension_codec;
mod expression_deduplication;
diff --git a/datafusion-examples/examples/query_planning/main.rs b/datafusion-examples/examples/query_planning/main.rs
index ec21c3ea5a..d3f99aedce 100644
--- a/datafusion-examples/examples/query_planning/main.rs
+++ b/datafusion-examples/examples/query_planning/main.rs
@@ -26,14 +26,30 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `analyzer_rule` — use a custom AnalyzerRule to change a query's semantics (row level access control)
-//! - `expr_api` — create, execute, simplify, analyze and coerce `Expr`s
-//! - `optimizer_rule` — use a custom OptimizerRule to replace certain predicates
-//! - `parse_sql_expr` — parse SQL text into DataFusion `Expr`
-//! - `plan_to_sql` — generate SQL from DataFusion `Expr` and `LogicalPlan`
-//! - `planner_api` — APIs to manipulate logical and physical plans
-//! - `pruning` — APIs to manipulate logical and physical plans
-//! - `thread_pools` — demonstrate TrackConsumersPool for memory tracking and debugging with enhanced error messages and shows how to implement memory-aware ExecutionPlan with memory reservation and spilling
+//!
+//! - `analyzer_rule`
+//! (file: analyzer_rule.rs, desc: Custom AnalyzerRule to change query semantics)
+//!
+//! - `expr_api`
+//! (file: expr_api.rs, desc: Create, execute, analyze, and coerce Exprs)
+//!
+//! - `optimizer_rule`
+//! (file: optimizer_rule.rs, desc: Replace predicates via a custom OptimizerRule)
+//!
+//! - `parse_sql_expr`
+//! (file: parse_sql_expr.rs, desc: Parse SQL into DataFusion Expr)
+//!
+//! - `plan_to_sql`
+//! (file: plan_to_sql.rs, desc: Generate SQL from expressions or plans)
+//!
+//! - `planner_api`
+//! (file: planner_api.rs, desc: APIs for logical and physical plan manipulation)
+//!
+//! - `pruning`
+//! (file: pruning.rs, desc: Use pruning to skip irrelevant files)
+//!
+//! - `thread_pools`
+//! (file: thread_pools.rs, desc: Configure custom thread pools for DataFusion execution)
mod analyzer_rule;
mod expr_api;
diff --git a/datafusion-examples/examples/relation_planner/main.rs b/datafusion-examples/examples/relation_planner/main.rs
index 15079f6446..babc0d3714 100644
--- a/datafusion-examples/examples/relation_planner/main.rs
+++ b/datafusion-examples/examples/relation_planner/main.rs
@@ -27,9 +27,15 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `match_recognize` — MATCH_RECOGNIZE pattern matching on event streams
-//! - `pivot_unpivot` — PIVOT and UNPIVOT operations for reshaping data
-//! - `table_sample` — TABLESAMPLE clause for sampling rows from tables
+//!
+//! - `match_recognize`
+//! (file: match_recognize.rs, desc: Implement MATCH_RECOGNIZE pattern matching)
+//!
+//! - `pivot_unpivot`
+//! (file: pivot_unpivot.rs, desc: Implement PIVOT / UNPIVOT)
+//!
+//! - `table_sample`
+//! (file: table_sample.rs, desc: Implement TABLESAMPLE)
//!
//! ## Snapshot Testing
//!
diff --git a/datafusion-examples/examples/sql_ops/main.rs b/datafusion-examples/examples/sql_ops/main.rs
index aaab7778be..ce7be8fa2b 100644
--- a/datafusion-examples/examples/sql_ops/main.rs
+++ b/datafusion-examples/examples/sql_ops/main.rs
@@ -26,10 +26,18 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `analysis` — analyse SQL queries with DataFusion structures
-//! - `custom_sql_parser` — implementing a custom SQL parser to extend DataFusion
-//! - `frontend` — create LogicalPlans (only) from sql strings
-//! - `query` — query data using SQL (in memory RecordBatches, local Parquet files)
+//!
+//! - `analysis`
+//! (file: analysis.rs, desc: Analyze SQL queries)
+//!
+//! - `custom_sql_parser`
+//! (file: custom_sql_parser.rs, desc: Implement a custom SQL parser to extend DataFusion)
+//!
+//! - `frontend`
+//! (file: frontend.rs, desc: Build LogicalPlans from SQL)
+//!
+//! - `query`
+//! (file: query.rs, desc: Query data using SQL)
mod analysis;
mod custom_sql_parser;
diff --git a/datafusion-examples/examples/udf/main.rs b/datafusion-examples/examples/udf/main.rs
index aff20e7754..e024e466ab 100644
--- a/datafusion-examples/examples/udf/main.rs
+++ b/datafusion-examples/examples/udf/main.rs
@@ -26,14 +26,30 @@
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
-//! - `adv_udaf` — user defined aggregate function example
-//! - `adv_udf` — user defined scalar function example
-//! - `adv_udwf` — user defined window function example
-//! - `async_udf` — asynchronous user defined function example
-//! - `udaf` — simple user defined aggregate function example
-//! - `udf` — simple user defined scalar function example
-//! - `udtf` — simple user defined table function example
-//! - `udwf` — simple user defined window function example
+//!
+//! - `adv_udaf`
+//! (file: advanced_udaf.rs, desc: Advanced User Defined Aggregate Function (UDAF))
+//!
+//! - `adv_udf`
+//! (file: advanced_udf.rs, desc: Advanced User Defined Scalar Function (UDF))
+//!
+//! - `adv_udwf`
+//! (file: advanced_udwf.rs, desc: Advanced User Defined Window Function (UDWF))
+//!
+//! - `async_udf`
+//! (file: async_udf.rs, desc: Asynchronous User Defined Scalar Function)
+//!
+//! - `udaf`
+//! (file: simple_udaf.rs, desc: Simple UDAF example)
+//!
+//! - `udf`
+//! (file: simple_udf.rs, desc: Simple UDF example)
+//!
+//! - `udtf`
+//! (file: simple_udtf.rs, desc: Simple UDTF example)
+//!
+//! - `udwf`
+//! (file: simple_udwf.rs, desc: Simple UDWF example)
mod advanced_udaf;
mod advanced_udf;
diff --git a/datafusion-examples/src/bin/examples-docs.rs b/datafusion-examples/src/bin/examples-docs.rs
new file mode 100644
index 0000000000..09a07e5fcd
--- /dev/null
+++ b/datafusion-examples/src/bin/examples-docs.rs
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Generates Markdown documentation for DataFusion example groups.
+//!
+//! This binary scans `datafusion-examples/examples`, extracts structured
+//! documentation from each group's `main.rs` file, and renders a README-style
+//! Markdown document.
+//!
+//! By default, documentation is generated for all example groups. If a group
+//! name is provided as the first CLI argument, only that group is rendered.
+//!
+//! ## Usage
+//!
+//! ```bash
+//! # Generate docs for all example groups
+//! cargo run --bin examples-docs
+//!
+//! # Generate docs for a single group
+//! cargo run --bin examples-docs -- dataframe
+//! ```
+
+use datafusion_examples::utils::examples_docs::{RepoLayout, generate_examples_readme};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let layout = RepoLayout::detect()?;
+ let group = std::env::args().nth(1);
+ let markdown = generate_examples_readme(&layout, group.as_deref())?;
+ print!("{markdown}");
+ Ok(())
+}
diff --git a/datafusion-examples/src/utils/examples_docs.rs b/datafusion-examples/src/utils/examples_docs.rs
new file mode 100644
index 0000000000..0efa4928c0
--- /dev/null
+++ b/datafusion-examples/src/utils/examples_docs.rs
@@ -0,0 +1,684 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Documentation generator for DataFusion examples.
+//!
+//! # Design goals
+//!
+//! - Keep README.md in sync with runnable examples
+//! - Fail fast on malformed documentation
+//!
+//! # Overview
+//!
+//! Each example group corresponds to a directory under
+//! `datafusion-examples/examples/<group>` containing a `main.rs` file.
+//! Documentation is extracted from structured `//!` comments in that file.
+//!
+//! For each example group, the generator produces:
+//!
+//! ```text
+//! ## <Group Name> Examples
+//! ### Group: `<group>`
+//! #### Category: Single Process | Distributed
+//!
+//! | Subcommand | File Path | Description |
+//! ```
+//!
+//! # Usage
+//!
+//! Generate documentation for a single group only:
+//!
+//! ```bash
+//! cargo run --bin examples-docs -- dataframe
+//! ```
+//!
+//! Generate documentation for all examples:
+//!
+//! ```bash
+//! cargo run --bin examples-docs
+//! ```
+
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use datafusion::error::{DataFusionError, Result};
+
+const STATIC_HEADER: &str = r#"<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# DataFusion Examples
+
+This crate includes end to end, highly commented examples of how to use
+various DataFusion APIs to help you get started.
+
+## Prerequisites
+
+Run `git submodule update --init` to init test files.
+
+## Running Examples
+
+To run an example, use the `cargo run` command, such as:
+
+```bash
+git clone https://github.com/apache/datafusion
+cd datafusion
+# Download test data
+git submodule update --init
+
+# Change to the examples directory
+cd datafusion-examples/examples
+
+# Run all examples in a group
+cargo run --example <group> -- all
+
+# Run a specific example within a group
+cargo run --example <group> -- <subcommand>
+
+# Run all examples in the `dataframe` group
+cargo run --example dataframe -- all
+
+# Run a single example from the `dataframe` group
+# (apply the same pattern for any other group)
+cargo run --example dataframe -- dataframe
+```
+"#;
+
+const ABBREVIATIONS: &[(&str, &str)] = &[
+ ("dataframe", "DataFrame"),
+ ("io", "IO"),
+ ("sql", "SQL"),
+ ("udf", "UDF"),
+];
+
+/// Describes the layout of a DataFusion repository.
+///
+/// This type centralizes knowledge about where example-related
+/// directories live relative to the repository root.
+#[derive(Debug, Clone)]
+pub struct RepoLayout {
+ root: PathBuf,
+}
+
+impl From<&Path> for RepoLayout {
+ fn from(path: &Path) -> Self {
+ Self {
+ root: path.to_path_buf(),
+ }
+ }
+}
+
+impl RepoLayout {
+ /// Creates a layout from an explicit repository root.
+ pub fn from_root(root: PathBuf) -> Self {
+ Self { root }
+ }
+
+ /// Detects the repository root based on `CARGO_MANIFEST_DIR`.
+ ///
+ /// This is intended for use from binaries inside the workspace.
+ pub fn detect() -> Result<Self> {
+ let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+
+ let root = manifest_dir.parent().ok_or_else(|| {
+ DataFusionError::Execution(
+ "CARGO_MANIFEST_DIR does not have a parent".to_string(),
+ )
+ })?;
+
+ Ok(Self {
+ root: root.to_path_buf(),
+ })
+ }
+
+ /// Returns the repository root directory.
+ pub fn root(&self) -> &Path {
+ &self.root
+ }
+
+ /// Returns the `datafusion-examples/examples` directory.
+ pub fn examples_root(&self) -> PathBuf {
+ self.root.join("datafusion-examples").join("examples")
+ }
+
+ /// Returns the directory for a single example group.
+ ///
+ /// Example: `examples/udf`
+ pub fn example_group_dir(&self, group: &str) -> PathBuf {
+ self.examples_root().join(group)
+ }
+}
+
+/// A group of related examples (e.g. `builtin_functions`, `udf`).
+///
+/// Each group corresponds to a directory containing a `main.rs` file
+/// with structured documentation comments.
+#[derive(Debug)]
+pub struct ExampleGroup {
+ pub name: GroupName,
+ pub examples: Vec<ExampleEntry>,
+ pub category: Category,
+}
+
+impl ExampleGroup {
+ /// Parses an example group from its directory.
+ ///
+ /// The group name is derived from the directory name, and example
+ /// entries are extracted from `main.rs`.
+ pub fn from_dir(dir: &Path, category: Category) -> Result<Self> {
+ let raw_name = dir
+ .file_name()
+ .and_then(|s| s.to_str())
+ .ok_or_else(|| {
+ DataFusionError::Execution("Invalid example group dir".to_string())
+ })?
+ .to_string();
+
+ let name = GroupName::from_dir_name(raw_name);
+ let main_rs = dir.join("main.rs");
+ let examples = parse_main_rs_docs(&main_rs)?;
+
+ Ok(Self {
+ name,
+ examples,
+ category,
+ })
+ }
+
+ // Renders this example group as a Markdown section.
+ pub fn render_markdown(&self) -> String {
+ let mut out = String::new();
+ out.push_str(&format!("\n## {} Examples\n\n", self.name.title()));
+ out.push_str(&format!("### Group: `{}`\n\n", self.name.raw()));
+ out.push_str(&format!("#### Category: {}\n\n", self.category.name()));
+ out.push_str("| Subcommand | File Path | Description |\n");
+ out.push_str("| --- | --- | --- |\n");
+
+ for ex in &self.examples {
+ out.push_str(&format!(
+ "| {} | [`{}/{}`](examples/{}/{}) | {} |\n",
+ ex.subcommand,
+ self.name.raw(),
+ ex.file,
+ self.name.raw(),
+ ex.file,
+ ex.desc
+ ));
+ }
+
+ out
+ }
+}
+
+/// Represents an example group name in both raw and human-readable forms.
+///
+/// For example:
+/// - raw: `builtin_functions`
+/// - title: `Builtin Functions`
+#[derive(Debug)]
+pub struct GroupName {
+ raw: String,
+ title: String,
+}
+
+impl GroupName {
+ /// Creates a group name from a directory name.
+ pub fn from_dir_name(raw: String) -> Self {
+ let title = raw
+ .split('_')
+ .map(format_part)
+ .collect::<Vec<_>>()
+ .join(" ");
+
+ Self { raw, title }
+ }
+
+ /// Returns the raw group name (directory name).
+ pub fn raw(&self) -> &str {
+ &self.raw
+ }
+
+ /// Returns a title-cased name for documentation.
+ pub fn title(&self) -> &str {
+ &self.title
+ }
+}
+
+/// A single runnable example within a group.
+///
+/// Each entry corresponds to a subcommand documented in `main.rs`.
+#[derive(Debug)]
+pub struct ExampleEntry {
+ /// CLI subcommand name.
+ pub subcommand: String,
+ /// Rust source file name.
+ pub file: String,
+ /// Human-readable description.
+ pub desc: String,
+}
+
+/// Execution category of an example group.
+#[derive(Debug, Default)]
+pub enum Category {
+ /// Runs in a single process.
+ #[default]
+ SingleProcess,
+ /// Requires a distributed setup.
+ Distributed,
+}
+
+impl Category {
+ /// Returns the display name used in documentation.
+ pub fn name(&self) -> &str {
+ match self {
+ Self::SingleProcess => "Single Process",
+ Self::Distributed => "Distributed",
+ }
+ }
+
+ /// Determines the category for a group by name.
+ pub fn for_group(name: &str) -> Self {
+ match name {
+ "flight" => Category::Distributed,
+ _ => Category::SingleProcess,
+ }
+ }
+}
+
+/// Generates Markdown documentation for DataFusion examples.
+///
+/// If `group` is `None`, documentation is generated for all example groups.
+/// If `group` is `Some`, only that group is rendered.
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - the requested group does not exist
+/// - a `main.rs` file is missing
+/// - documentation comments are malformed
+pub fn generate_examples_readme(
+ layout: &RepoLayout,
+ group: Option<&str>,
+) -> Result<String> {
+ let examples_root = layout.examples_root();
+
+ let mut out = String::new();
+ out.push_str(STATIC_HEADER);
+
+ let group_dirs: Vec<PathBuf> = match group {
+ Some(name) => {
+ let dir = examples_root.join(name);
+ if !dir.is_dir() {
+ return Err(DataFusionError::Execution(format!(
+ "Example group `{name}` does not exist"
+ )));
+ }
+ vec![dir]
+ }
+ None => discover_example_groups(&examples_root)?,
+ };
+
+ for group_dir in group_dirs {
+ let raw_name =
+ group_dir
+ .file_name()
+ .and_then(|s| s.to_str())
+ .ok_or_else(|| {
+ DataFusionError::Execution("Invalid example group dir".to_string())
+ })?;
+
+ let category = Category::for_group(raw_name);
+ let group = ExampleGroup::from_dir(&group_dir, category)?;
+
+ out.push_str(&group.render_markdown());
+ }
+
+ Ok(out)
+}
+
+/// Parses example entries from a group's `main.rs` file.
+pub fn parse_main_rs_docs(path: &Path) -> Result<Vec<ExampleEntry>> {
+ let content = fs::read_to_string(path)?;
+ let mut entries = Vec::new();
+ let mut pending_subcommand: Option<String> = None;
+
+ for raw_line in content.lines() {
+ let line = raw_line.trim();
+
+ if let Some(sub) = parse_subcommand_line(line) {
+ if sub != "all" {
+ pending_subcommand = Some(sub);
+ }
+ continue;
+ }
+
+ if let Some((file, desc)) = parse_metadata_line(line) {
+ let subcommand = pending_subcommand.take().ok_or_else(|| {
+ DataFusionError::Execution(
+ "Metadata without preceding subcommand".to_string(),
+ )
+ })?;
+
+ entries.push(ExampleEntry {
+ subcommand,
+ file,
+ desc,
+ });
+ }
+ }
+
+ Ok(entries)
+}
+
+/// Parses a subcommand declaration line from `main.rs` docs.
+///
+/// Expected format:
+/// ```text
+/// //! - `<subcommand>`
+/// ```
+fn parse_subcommand_line(line: &str) -> Option<String> {
+ line.strip_prefix("//! - `")
+ .and_then(|rest| rest.strip_suffix('`'))
+ .map(|s| s.to_string())
+}
+
+/// Parses example metadata (file name and description) from `main.rs` docs.
+///
+/// Expected format:
+/// ```text
+/// //! (file: <file>.rs, desc: <description>)
+/// ```
+fn parse_metadata_line(line: &str) -> Option<(String, String)> {
+ let line = line.strip_prefix("//!").map(str::trim)?;
+ if !line.starts_with("(file:") || !line.ends_with(')') {
+ return None;
+ }
+ let inner = line.strip_prefix('(')?.strip_suffix(')')?;
+ let (file_part, desc_part) = inner.split_once(", desc:")?;
+ let file = file_part.strip_prefix("file:")?.trim().to_string();
+ let desc = desc_part.trim().to_string();
+ Some((file, desc))
+}
+
+/// Discovers all example group directories under the given root.
+fn discover_example_groups(root: &Path) -> Result<Vec<PathBuf>> {
+ let mut groups = Vec::new();
+ for entry in fs::read_dir(root)? {
+ let entry = entry?;
+ let path = entry.path();
+
+ if path.is_dir() && path.join("main.rs").exists() {
+ groups.push(path);
+ }
+ }
+
+ groups.sort();
+
+ Ok(groups)
+}
+
+/// Formats a single group-name segment for display.
+///
+/// This function applies DataFusion-specific capitalization rules:
+/// - Known abbreviations (e.g. `sql`, `io`, `udf`) are rendered in all caps
+/// - All other segments fall back to standard Title Case
+fn format_part(part: &str) -> String {
+ let lower = part.to_ascii_lowercase();
+
+ if let Some((_, replacement)) = ABBREVIATIONS.iter().find(|(k, _)| *k == lower) {
+ return replacement.to_string();
+ }
+
+ let mut chars = part.chars();
+ match chars.next() {
+ Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
+ None => String::new(),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use tempfile::TempDir;
+
+ #[test]
+ fn all_subcommand_is_ignored() -> Result<()> {
+ let tmp = TempDir::new().unwrap();
+ let group_dir = tmp.path().join("foo");
+ fs::create_dir(&group_dir)?;
+
+ fs::write(
+ group_dir.join("main.rs"),
+ r#"
+//! - `all`
+//!
+//! - `foo`
+//! (file: foo.rs, desc: foo example)
+"#,
+ )?;
+
+ let group = ExampleGroup::from_dir(&group_dir, Category::SingleProcess)?;
+
+ assert_eq!(group.examples.len(), 1);
+ assert_eq!(group.examples[0].subcommand, "foo");
+
+ Ok(())
+ }
+
+ #[test]
+ fn parse_subcommand_line_works() {
+ let line = "//! - `date_time`";
+ let sub = parse_subcommand_line(line).unwrap();
+ assert_eq!(sub, "date_time");
+ }
+
+ #[test]
+ fn parse_metadata_line_works() {
+ let line =
+ "//! (file: date_time.rs, desc: Examples of date-time related functions)";
+ let (file, desc) = parse_metadata_line(line).unwrap();
+ assert_eq!(file, "date_time.rs");
+ assert_eq!(desc, "Examples of date-time related functions");
+
+ let line = "//! (file: foo.rs, desc: Foo, bar, baz)";
+ let (file, desc) = parse_metadata_line(line).unwrap();
+ assert_eq!(file, "foo.rs");
+ assert_eq!(desc, "Foo, bar, baz");
+ }
+
+ #[test]
+ fn metadata_without_subcommand_fails() {
+ let tmp = TempDir::new().unwrap();
+ let group_dir = tmp.path().join("bad");
+ fs::create_dir(&group_dir).unwrap();
+
+ fs::write(
+ group_dir.join("main.rs"),
+ "//! (file: foo.rs, desc: missing subcommand)",
+ )
+ .unwrap();
+
+ let err =
+ ExampleGroup::from_dir(&group_dir, Category::SingleProcess).unwrap_err();
+
+ assert!(
+ err.to_string()
+ .contains("Metadata without preceding subcommand"),
+ "unexpected error: {err}"
+ );
+ }
+
+ #[test]
+ fn parse_group_example_works() -> Result<()> {
+ let tmp = TempDir::new().unwrap();
+
+ // Simulate: examples/builtin_functions/
+ let group_dir = tmp.path().join("builtin_functions");
+ fs::create_dir(&group_dir)?;
+
+ // Write a fake main.rs with docs
+ let main_rs = group_dir.join("main.rs");
+ fs::write(
+ &main_rs,
+ r#"
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+//! # These are miscellaneous function-related examples
+//!
+//! These examples demonstrate miscellaneous function-related features.
+//!
+//! ## Usage
+//! ```bash
+//! cargo run --example builtin_functions -- [all|date_time|function_factory|regexp]
+//! ```
+//!
+//! Each subcommand runs a corresponding example:
+//! - `all` — run all examples included in this module
+//!
+//! - `date_time`
+//! (file: date_time.rs, desc: Examples of date-time related functions and queries)
+//!
+//! - `function_factory`
+//! (file: function_factory.rs, desc: Register `CREATE FUNCTION` handler to implement SQL macros)
+//!
+//! - `regexp`
+//! (file: regexp.rs, desc: Examples of using regular expression functions)
+"#,
+ )?;
+
+ let group = ExampleGroup::from_dir(&group_dir, Category::SingleProcess)?;
+
+ // Assert group-level data
+ assert_eq!(group.name.title(), "Builtin Functions");
+ assert_eq!(group.examples.len(), 3);
+
+ // Assert 1 example
+ assert_eq!(group.examples[0].subcommand, "date_time");
+ assert_eq!(group.examples[0].file, "date_time.rs");
+ assert_eq!(
+ group.examples[0].desc,
+ "Examples of date-time related functions and queries"
+ );
+
+ // Assert 2 example
+ assert_eq!(group.examples[1].subcommand, "function_factory");
+ assert_eq!(group.examples[1].file, "function_factory.rs");
+ assert_eq!(
+ group.examples[1].desc,
+ "Register `CREATE FUNCTION` handler to implement SQL macros"
+ );
+
+ // Assert 3 example
+ assert_eq!(group.examples[2].subcommand, "regexp");
+ assert_eq!(group.examples[2].file, "regexp.rs");
+ assert_eq!(
+ group.examples[2].desc,
+ "Examples of using regular expression functions"
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn single_group_generation_works() {
+ let tmp = TempDir::new().unwrap();
+ // Fake repo root
+ let layout = RepoLayout::from_root(tmp.path().to_path_buf());
+
+ // Create: datafusion-examples/examples/builtin_functions
+ let examples_dir = layout.example_group_dir("builtin_functions");
+ fs::create_dir_all(&examples_dir).unwrap();
+
+ fs::write(
+ examples_dir.join("main.rs"),
+ "//! - `x`\n//! (file: foo.rs, desc: test)",
+ )
+ .unwrap();
+
+ let out = generate_examples_readme(&layout, Some("builtin_functions")).unwrap();
+
+ assert!(out.contains("Builtin Functions"));
+ }
+
+ #[test]
+ fn single_group_generation_fails_if_group_missing() {
+ let tmp = TempDir::new().unwrap();
+ let layout = RepoLayout::from_root(tmp.path().to_path_buf());
+
+ let err = generate_examples_readme(&layout, Some("missing_group")).unwrap_err();
+
+ assert!(
+ err.to_string()
+ .contains("Example group `missing_group` does not exist"),
+ "unexpected error: {err}"
+ );
+ }
+
+ #[test]
+ fn group_name_title_is_human_readable() {
+ let name = GroupName::from_dir_name("very_long_group_name".to_string());
+ assert_eq!(name.title(), "Very Long Group Name");
+ }
+
+ #[test]
+ fn group_name_handles_abbreviations() {
+ assert_eq!(
+ GroupName::from_dir_name("dataframe".to_string()).title(),
+ "DataFrame"
+ );
+ assert_eq!(
+ GroupName::from_dir_name("data_io".to_string()).title(),
+ "Data IO"
+ );
+ assert_eq!(
+ GroupName::from_dir_name("sql_ops".to_string()).title(),
+ "SQL Ops"
+ );
+ assert_eq!(GroupName::from_dir_name("udf".to_string()).title(), "UDF");
+ }
+}
diff --git a/datafusion-examples/src/utils/mod.rs b/datafusion-examples/src/utils/mod.rs
index b9e5b487db..aa44491296 100644
--- a/datafusion-examples/src/utils/mod.rs
+++ b/datafusion-examples/src/utils/mod.rs
@@ -17,5 +17,6 @@
mod csv_to_parquet;
pub mod datasets;
+pub mod examples_docs;
pub use csv_to_parquet::write_csv_to_parquet;