Apache DataFusion SQL Query Engine
Refactor example metadata parsing utilities(#20204) (#20233)
Sergey Zhukov5844dfb349d9
diff --git a/datafusion-examples/src/bin/examples-docs.rs b/datafusion-examples/src/bin/examples-docs.rsindex 09a07e5fcd..7efcf4da15 100644--- a/datafusion-examples/src/bin/examples-docs.rs+++ b/datafusion-examples/src/bin/examples-docs.rs@@ -34,7 +34,9 @@//! cargo run --bin examples-docs -- dataframe//! ```-use datafusion_examples::utils::examples_docs::{RepoLayout, generate_examples_readme};+use datafusion_examples::utils::example_metadata::{+ RepoLayout, generate_examples_readme,+};fn main() -> Result<(), Box<dyn std::error::Error>> {let layout = RepoLayout::detect()?;diff --git a/datafusion-examples/src/utils/example_metadata/discover.rs b/datafusion-examples/src/utils/example_metadata/discover.rsnew file mode 100644index 0000000000..c5cf3ec111--- /dev/null+++ b/datafusion-examples/src/utils/example_metadata/discover.rs@@ -0,0 +1,75 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Utilities for discovering example groups in the repository filesystem.+//!+//! An example group is defined as a directory containing a `main.rs` file+//! under the examples root. This module is intentionally filesystem-focused+//! and does not perform any parsing or rendering.++use std::fs;+use std::path::{Path, PathBuf};++use datafusion::error::Result;++/// Discovers all example group directories under the given root.+///+/// A directory is considered an example group if it contains a `main.rs` file.+pub fn discover_example_groups(root: &Path) -> Result<Vec<PathBuf>> {+ let mut groups = Vec::new();+ for entry in fs::read_dir(root)? {+ let entry = entry?;+ let path = entry.path();++ if path.is_dir() && path.join("main.rs").exists() {+ groups.push(path);+ }+ }+ groups.sort();+ Ok(groups)+}++#[cfg(test)]+mod tests {+ use super::*;++ use std::fs::{self, File};++ use tempfile::TempDir;++ #[test]+ fn discover_example_groups_finds_dirs_with_main_rs() -> Result<()> {+ let tmp = TempDir::new()?;+ let root = tmp.path();++ // valid example group+ let group1 = root.join("group1");+ fs::create_dir(&group1)?;+ File::create(group1.join("main.rs"))?;++ // not an example group+ let group2 = root.join("group2");+ fs::create_dir(&group2)?;++ let groups = discover_example_groups(root)?;++ assert_eq!(groups.len(), 1);+ assert_eq!(groups[0], group1);++ Ok(())+ }+}diff --git a/datafusion-examples/src/utils/example_metadata/layout.rs b/datafusion-examples/src/utils/example_metadata/layout.rsnew file mode 100644index 0000000000..ee6fad8985--- /dev/null+++ b/datafusion-examples/src/utils/example_metadata/layout.rs@@ -0,0 +1,113 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Repository layout utilities.+//!+//! This module provides a small helper (`RepoLayout`) that encapsulates+//! knowledge about the DataFusion repository structure, in particular+//! where example groups are located relative to the repository root.++use std::path::{Path, PathBuf};++use datafusion::error::{DataFusionError, Result};++/// Describes the layout of a DataFusion repository.+///+/// This type centralizes knowledge about where example-related+/// directories live relative to the repository root.+#[derive(Debug, Clone)]+pub struct RepoLayout {+ root: PathBuf,+}++impl From<&Path> for RepoLayout {+ fn from(path: &Path) -> Self {+ Self {+ root: path.to_path_buf(),+ }+ }+}++impl RepoLayout {+ /// Creates a layout from an explicit repository root.+ pub fn from_root(root: PathBuf) -> Self {+ Self { root }+ }++ /// Detects the repository root based on `CARGO_MANIFEST_DIR`.+ ///+ /// This is intended for use from binaries inside the workspace.+ pub fn detect() -> Result<Self> {+ let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));++ let root = manifest_dir.parent().ok_or_else(|| {+ DataFusionError::Execution(+ "CARGO_MANIFEST_DIR does not have a parent".to_string(),+ )+ })?;++ Ok(Self {+ root: root.to_path_buf(),+ })+ }++ /// Returns the repository root directory.+ pub fn root(&self) -> &Path {+ &self.root+ }++ /// Returns the `datafusion-examples/examples` directory.+ pub fn examples_root(&self) -> PathBuf {+ self.root.join("datafusion-examples").join("examples")+ }++ /// Returns the directory for a single example group.+ ///+ /// Example: `examples/udf`+ pub fn example_group_dir(&self, group: &str) -> PathBuf {+ self.examples_root().join(group)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn detect_sets_non_empty_root() -> Result<()> {+ let layout = RepoLayout::detect()?;+ assert!(!layout.root().as_os_str().is_empty());+ Ok(())+ }++ #[test]+ fn examples_root_is_under_repo_root() -> Result<()> {+ let layout = RepoLayout::detect()?;+ let examples_root = layout.examples_root();+ assert!(examples_root.starts_with(layout.root()));+ assert!(examples_root.ends_with("datafusion-examples/examples"));+ Ok(())+ }++ #[test]+ fn example_group_dir_appends_group_name() -> Result<()> {+ let layout = RepoLayout::detect()?;+ let group_dir = layout.example_group_dir("foo");+ assert!(group_dir.ends_with("datafusion-examples/examples/foo"));+ Ok(())+ }+}diff --git a/datafusion-examples/src/utils/example_metadata/mod.rs b/datafusion-examples/src/utils/example_metadata/mod.rsnew file mode 100644index 0000000000..ab4c8e4a8e--- /dev/null+++ b/datafusion-examples/src/utils/example_metadata/mod.rs@@ -0,0 +1,67 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Documentation generator for DataFusion examples.+//!+//! # Design goals+//!+//! - Keep README.md in sync with runnable examples+//! - Fail fast on malformed documentation+//!+//! # Overview+//!+//! Each example group corresponds to a directory under+//! `datafusion-examples/examples/<group>` containing a `main.rs` file.+//! Documentation is extracted from structured `//!` comments in that file.+//!+//! For each example group, the generator produces:+//!+//! ```text+//! ## <Group Name> Examples+//! ### Group: `<group>`+//! #### Category: Single Process | Distributed+//!+//! | Subcommand | File Path | Description |+//! ```+//!+//! # Usage+//!+//! Generate documentation for a single group only:+//!+//! ```bash+//! cargo run --bin examples-docs -- dataframe+//! ```+//!+//! Generate documentation for all examples:+//!+//! ```bash+//! cargo run --bin examples-docs+//! ```++pub mod discover;+pub mod layout;+pub mod model;+pub mod parser;+pub mod render;++#[cfg(test)]+pub mod test_utils;++pub use layout::RepoLayout;+pub use model::{Category, ExampleEntry, ExampleGroup, GroupName};+pub use parser::parse_main_rs_docs;+pub use render::generate_examples_readme;diff --git a/datafusion-examples/src/utils/example_metadata/model.rs b/datafusion-examples/src/utils/example_metadata/model.rsnew file mode 100644index 0000000000..74b605cde1--- /dev/null+++ b/datafusion-examples/src/utils/example_metadata/model.rs@@ -0,0 +1,409 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Domain model for DataFusion example documentation.+//!+//! This module defines the core data structures used to represent+//! example groups, individual examples, and their categorization+//! as parsed from `main.rs` documentation comments.++use std::path::Path;++use datafusion::error::{DataFusionError, Result};++use crate::utils::example_metadata::{parse_main_rs_docs, render::ABBREVIATIONS};++/// A group of related examples (e.g. `builtin_functions`, `udf`).+///+/// Each group corresponds to a directory containing a `main.rs` file+/// with structured documentation comments.+#[derive(Debug)]+pub struct ExampleGroup {+ pub name: GroupName,+ pub examples: Vec<ExampleEntry>,+ pub category: Category,+}++impl ExampleGroup {+ /// Parses an example group from its directory.+ ///+ /// The group name is derived from the directory name, and example+ /// entries are extracted from `main.rs`.+ pub fn from_dir(dir: &Path, category: Category) -> Result<Self> {+ let raw_name = dir+ .file_name()+ .and_then(|s| s.to_str())+ .ok_or_else(|| {+ DataFusionError::Execution("Invalid example group dir".to_string())+ })?+ .to_string();++ let name = GroupName::from_dir_name(raw_name);+ let main_rs = dir.join("main.rs");+ let examples = parse_main_rs_docs(&main_rs)?;++ Ok(Self {+ name,+ examples,+ category,+ })+ }+}++/// Represents an example group name in both raw and human-readable forms.+///+/// For example:+/// - raw: `builtin_functions`+/// - title: `Builtin Functions`+#[derive(Debug)]+pub struct GroupName {+ raw: String,+ title: String,+}++impl GroupName {+ /// Creates a group name from a directory name.+ pub fn from_dir_name(raw: String) -> Self {+ let title = raw+ .split('_')+ .map(format_part)+ .collect::<Vec<_>>()+ .join(" ");++ Self { raw, title }+ }++ /// Returns the raw group name (directory name).+ pub fn raw(&self) -> &str {+ &self.raw+ }++ /// Returns a title-cased name for documentation.+ pub fn title(&self) -> &str {+ &self.title+ }+}++/// A single runnable example within a group.+///+/// Each entry corresponds to a subcommand documented in `main.rs`.+#[derive(Debug)]+pub struct ExampleEntry {+ /// CLI subcommand name.+ pub subcommand: String,+ /// Rust source file name.+ pub file: String,+ /// Human-readable description.+ pub desc: String,+}++/// Execution category of an example group.+#[derive(Debug, Default)]+pub enum Category {+ /// Runs in a single process.+ #[default]+ SingleProcess,+ /// Requires a distributed setup.+ Distributed,+}++impl Category {+ /// Returns the display name used in documentation.+ pub fn name(&self) -> &str {+ match self {+ Self::SingleProcess => "Single Process",+ Self::Distributed => "Distributed",+ }+ }++ /// Determines the category for a group by name.+ pub fn for_group(name: &str) -> Self {+ match name {+ "flight" => Category::Distributed,+ _ => Category::SingleProcess,+ }+ }+}++/// Formats a single group-name segment for display.+///+/// This function applies DataFusion-specific capitalization rules:+/// - Known abbreviations (e.g. `sql`, `io`, `udf`) are rendered in all caps+/// - All other segments fall back to standard Title Case+fn format_part(part: &str) -> String {+ let lower = part.to_ascii_lowercase();++ if let Some((_, replacement)) = ABBREVIATIONS.iter().find(|(k, _)| *k == lower) {+ return replacement.to_string();+ }++ let mut chars = part.chars();+ match chars.next() {+ Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),+ None => String::new(),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ use crate::utils::example_metadata::test_utils::{+ assert_exec_err_contains, example_group_from_docs,+ };++ use std::fs;++ use tempfile::TempDir;++ #[test]+ fn category_for_group_works() {+ assert!(matches!(+ Category::for_group("flight"),+ Category::Distributed+ ));+ assert!(matches!(+ Category::for_group("anything_else"),+ Category::SingleProcess+ ));+ }++ #[test]+ fn all_subcommand_is_ignored() -> Result<()> {+ let group = example_group_from_docs(+ r#"+ //! - `all` — run all examples included in this module+ //!+ //! - `foo`+ //! (file: foo.rs, desc: foo example)+ "#,+ )?;+ assert_eq!(group.examples.len(), 1);+ assert_eq!(group.examples[0].subcommand, "foo");+ Ok(())+ }++ #[test]+ fn metadata_without_subcommand_fails() {+ let err = example_group_from_docs("//! (file: foo.rs, desc: missing subcommand)")+ .unwrap_err();+ assert_exec_err_contains(err, "Metadata without preceding subcommand");+ }++ #[test]+ fn group_name_handles_abbreviations() {+ assert_eq!(+ GroupName::from_dir_name("dataframe".to_string()).title(),+ "DataFrame"+ );+ assert_eq!(+ GroupName::from_dir_name("data_io".to_string()).title(),+ "Data IO"+ );+ assert_eq!(+ GroupName::from_dir_name("sql_ops".to_string()).title(),+ "SQL Ops"+ );+ assert_eq!(GroupName::from_dir_name("udf".to_string()).title(), "UDF");+ }++ #[test]+ fn group_name_title_cases() {+ let cases = [+ ("very_long_group_name", "Very Long Group Name"),+ ("foo", "Foo"),+ ("dataframe", "DataFrame"),+ ("data_io", "Data IO"),+ ("sql_ops", "SQL Ops"),+ ("udf", "UDF"),+ ];+ for (input, expected) in cases {+ let name = GroupName::from_dir_name(input.to_string());+ assert_eq!(name.title(), expected);+ }+ }++ #[test]+ fn parse_group_example_works() -> Result<()> {+ let tmp = TempDir::new().unwrap();++ // Simulate: examples/builtin_functions/+ let group_dir = tmp.path().join("builtin_functions");+ fs::create_dir(&group_dir)?;++ // Write a fake main.rs with docs+ let main_rs = group_dir.join("main.rs");+ fs::write(+ &main_rs,+ r#"+ // Licensed to the Apache Software Foundation (ASF) under one+ // or more contributor license agreements. See the NOTICE file+ // distributed with this work for additional information+ // regarding copyright ownership. The ASF licenses this file+ // to you under the Apache License, Version 2.0 (the+ // "License"); you may not use this file except in compliance+ // with the License. You may obtain a copy of the License at+ //+ // http://www.apache.org/licenses/LICENSE-2.0+ //+ // Unless required by applicable law or agreed to in writing,+ // software distributed under the License is distributed on an+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ // KIND, either express or implied. See the License for the+ // specific language governing permissions and limitations+ // under the License.+ //+ //! # These are miscellaneous function-related examples+ //!+ //! These examples demonstrate miscellaneous function-related features.+ //!+ //! ## Usage+ //! ```bash+ //! cargo run --example builtin_functions -- [all|date_time|function_factory|regexp]+ //! ```+ //!+ //! Each subcommand runs a corresponding example:+ //! - `all` — run all examples included in this module+ //!+ //! - `date_time`+ //! (file: date_time.rs, desc: Examples of date-time related functions and queries)+ //!+ //! - `function_factory`+ //! (file: function_factory.rs, desc: Register `CREATE FUNCTION` handler to implement SQL macros)+ //!+ //! - `regexp`+ //! (file: regexp.rs, desc: Examples of using regular expression functions)+ "#,+ )?;++ let group = ExampleGroup::from_dir(&group_dir, Category::SingleProcess)?;++ // Assert group-level data+ assert_eq!(group.name.title(), "Builtin Functions");+ assert_eq!(group.examples.len(), 3);++ // Assert 1 example+ assert_eq!(group.examples[0].subcommand, "date_time");+ assert_eq!(group.examples[0].file, "date_time.rs");+ assert_eq!(+ group.examples[0].desc,+ "Examples of date-time related functions and queries"+ );++ // Assert 2 example+ assert_eq!(group.examples[1].subcommand, "function_factory");+ assert_eq!(group.examples[1].file, "function_factory.rs");+ assert_eq!(+ group.examples[1].desc,+ "Register `CREATE FUNCTION` handler to implement SQL macros"+ );++ // Assert 3 example+ assert_eq!(group.examples[2].subcommand, "regexp");+ assert_eq!(group.examples[2].file, "regexp.rs");+ assert_eq!(+ group.examples[2].desc,+ "Examples of using regular expression functions"+ );++ Ok(())+ }++ #[test]+ fn duplicate_metadata_without_repeating_subcommand_fails() {+ let err = example_group_from_docs(+ r#"+ //! - `foo`+ //! (file: a.rs, desc: first)+ //! (file: b.rs, desc: second)+ "#,+ )+ .unwrap_err();+ assert_exec_err_contains(err, "Metadata without preceding subcommand");+ }++ #[test]+ fn duplicate_metadata_for_same_subcommand_fails() {+ let err = example_group_from_docs(+ r#"+ //! - `foo`+ //! (file: a.rs, desc: first)+ //!+ //! - `foo`+ //! (file: b.rs, desc: second)+ "#,+ )+ .unwrap_err();+ assert_exec_err_contains(err, "Duplicate metadata for subcommand `foo`");+ }++ #[test]+ fn metadata_must_follow_subcommand() {+ let err = example_group_from_docs(+ r#"+ //! - `foo`+ //! some unrelated comment+ //! (file: foo.rs, desc: test)+ "#,+ )+ .unwrap_err();+ assert_exec_err_contains(err, "Metadata without preceding subcommand");+ }++ #[test]+ fn preserves_example_order_from_main_rs() -> Result<()> {+ let group = example_group_from_docs(+ r#"+ //! - `second`+ //! (file: second.rs, desc: second example)+ //!+ //! - `first`+ //! (file: first.rs, desc: first example)+ //!+ //! - `third`+ //! (file: third.rs, desc: third example)+ "#,+ )?;++ let subcommands: Vec<&str> = group+ .examples+ .iter()+ .map(|e| e.subcommand.as_str())+ .collect();++ assert_eq!(+ subcommands,+ vec!["second", "first", "third"],+ "examples must preserve the order defined in main.rs"+ );++ Ok(())+ }++ #[test]+ fn metadata_can_follow_blank_doc_line() -> Result<()> {+ let group = example_group_from_docs(+ r#"+ //! - `foo`+ //!+ //! (file: foo.rs, desc: test)+ "#,+ )?;+ assert_eq!(group.examples.len(), 1);+ Ok(())+ }+}diff --git a/datafusion-examples/src/utils/example_metadata/parser.rs b/datafusion-examples/src/utils/example_metadata/parser.rsnew file mode 100644index 0000000000..83105e7d40--- /dev/null+++ b/datafusion-examples/src/utils/example_metadata/parser.rs@@ -0,0 +1,274 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Parser for example metadata embedded in `main.rs` documentation comments.+//!+//! This module scans `//!` doc comments to extract example subcommands+//! and their associated metadata (file name and description), enforcing+//! a strict ordering and structure to avoid ambiguous documentation.++use std::path::Path;+use std::{collections::HashSet, fs};++use datafusion_common::{DataFusionError, Result};+use nom::{+ IResult, Parser,+ bytes::complete::{tag, take_until, take_while},+ character::complete::multispace0,+ combinator::all_consuming,+ sequence::{delimited, preceded},+};++use crate::utils::example_metadata::ExampleEntry;++/// Parsing state machine used while scanning `main.rs` docs.+///+/// This makes the "subcommand - metadata" relationship explicit:+/// metadata is only valid immediately after a subcommand has been seen.+enum ParserState<'a> {+ /// Not currently expecting metadata.+ Idle,+ /// A subcommand was just parsed; the next valid metadata (if any)+ /// must belong to this subcommand.+ SeenSubcommand(&'a str),+}++/// Parses a subcommand declaration line from `main.rs` docs.+///+/// Expected format:+/// ```text+/// //! - `<subcommand>`+/// ```+fn parse_subcommand_line(input: &str) -> IResult<&str, &str> {+ let parser = preceded(+ multispace0,+ delimited(tag("//! - `"), take_until("`"), tag("`")),+ );+ all_consuming(parser).parse(input)+}++/// Parses example metadata (file name and description) from `main.rs` docs.+///+/// Expected format:+/// ```text+/// //! (file: <file>.rs, desc: <description>)+/// ```+fn parse_metadata_line(input: &str) -> IResult<&str, (&str, &str)> {+ let parser = preceded(+ multispace0,+ preceded(tag("//!"), preceded(multispace0, take_while(|_| true))),+ );+ let (rest, payload) = all_consuming(parser).parse(input)?;++ let content = payload+ .strip_prefix("(")+ .and_then(|s| s.strip_suffix(")"))+ .ok_or_else(|| {+ nom::Err::Error(nom::error::Error::new(payload, nom::error::ErrorKind::Tag))+ })?;++ let (file, desc) = content+ .strip_prefix("file:")+ .ok_or_else(|| {+ nom::Err::Error(nom::error::Error::new(payload, nom::error::ErrorKind::Tag))+ })?+ .split_once(", desc:")+ .ok_or_else(|| {+ nom::Err::Error(nom::error::Error::new(payload, nom::error::ErrorKind::Tag))+ })?;++ Ok((rest, (file.trim(), desc.trim())))+}++/// Parses example entries from a group's `main.rs` file.+pub fn parse_main_rs_docs(path: &Path) -> Result<Vec<ExampleEntry>> {+ let content = fs::read_to_string(path)?;+ let mut entries = vec![];+ let mut state = ParserState::Idle;+ let mut seen_subcommands = HashSet::new();++ for (line_no, raw_line) in content.lines().enumerate() {+ let line = raw_line.trim();++ // Try parsing subcommand, excluding `all` because it's not used in README+ if let Ok((_, sub)) = parse_subcommand_line(line) {+ state = if sub == "all" {+ ParserState::Idle+ } else {+ ParserState::SeenSubcommand(sub)+ };+ continue;+ }++ // Try parsing metadata+ if let Ok((_, (file, desc))) = parse_metadata_line(line) {+ let subcommand = match state {+ ParserState::SeenSubcommand(s) => s,+ ParserState::Idle => {+ return Err(DataFusionError::Execution(format!(+ "Metadata without preceding subcommand at {}:{}",+ path.display(),+ line_no + 1+ )));+ }+ };++ if !seen_subcommands.insert(subcommand) {+ return Err(DataFusionError::Execution(format!(+ "Duplicate metadata for subcommand `{subcommand}`"+ )));+ }++ entries.push(ExampleEntry {+ subcommand: subcommand.to_string(),+ file: file.to_string(),+ desc: desc.to_string(),+ });++ state = ParserState::Idle;+ continue;+ }++ // If a non-blank doc line interrupts a pending subcommand, reset the state+ if let ParserState::SeenSubcommand(_) = state+ && is_non_blank_doc_line(line)+ {+ state = ParserState::Idle;+ }+ }++ Ok(entries)+}++/// Returns `true` for non-blank Rust doc comment lines (`//!`).+///+/// Used to detect when a subcommand is interrupted by unrelated documentation,+/// so metadata is only accepted immediately after a subcommand (blank doc lines+/// are allowed in between).+fn is_non_blank_doc_line(line: &str) -> bool {+ line.starts_with("//!") && !line.trim_start_matches("//!").trim().is_empty()+}++#[cfg(test)]+mod tests {+ use super::*;++ use tempfile::TempDir;++ #[test]+ fn parse_subcommand_line_accepts_valid_input() {+ let line = "//! - `date_time`";+ let sub = parse_subcommand_line(line);+ assert_eq!(sub, Ok(("", "date_time")));+ }++ #[test]+ fn parse_subcommand_line_invalid_inputs() {+ let err_lines = [+ "//! - ",+ "//! - foo",+ "//! - `foo` bar",+ "//! --",+ "//!-",+ "//!--",+ "//!",+ "//",+ "/",+ "",+ ];+ for line in err_lines {+ assert!(+ parse_subcommand_line(line).is_err(),+ "expected error for input: {line}"+ );+ }+ }++ #[test]+ fn parse_metadata_line_accepts_valid_input() {+ let line =+ "//! (file: date_time.rs, desc: Examples of date-time related functions)";+ let res = parse_metadata_line(line);+ assert_eq!(+ res,+ Ok((+ "",+ ("date_time.rs", "Examples of date-time related functions")+ ))+ );++ let line = "//! (file: foo.rs, desc: Foo, bar, baz)";+ let res = parse_metadata_line(line);+ assert_eq!(res, Ok(("", ("foo.rs", "Foo, bar, baz"))));++ let line = "//! (file: foo.rs, desc: Foo(FOO))";+ let res = parse_metadata_line(line);+ assert_eq!(res, Ok(("", ("foo.rs", "Foo(FOO)"))));+ }++ #[test]+ fn parse_metadata_line_invalid_inputs() {+ let bad_lines = [+ "//! (file: foo.rs)",+ "//! (desc: missing file)",+ "//! file: foo.rs, desc: test",+ "//! file: foo.rs,desc: test",+ "//! (file: foo.rs desc: test)",+ "//! (file: foo.rs,desc: test)",+ "//! (desc: test, file: foo.rs)",+ "//! ()",+ "//! (file: foo.rs, desc: test) extra",+ "",+ ];+ for line in bad_lines {+ assert!(+ parse_metadata_line(line).is_err(),+ "expected error for input: {line}"+ );+ }+ }++ #[test]+ fn parse_main_rs_docs_extracts_entries() -> Result<()> {+ let tmp = TempDir::new().unwrap();+ let main_rs = tmp.path().join("main.rs");++ fs::write(+ &main_rs,+ r#"+ //! - `foo`+ //! (file: foo.rs, desc: first example)+ //!+ //! - `bar`+ //! (file: bar.rs, desc: second example)+ "#,+ )?;++ let entries = parse_main_rs_docs(&main_rs)?;++ assert_eq!(entries.len(), 2);++ assert_eq!(entries[0].subcommand, "foo");+ assert_eq!(entries[0].file, "foo.rs");+ assert_eq!(entries[0].desc, "first example");++ assert_eq!(entries[1].subcommand, "bar");+ assert_eq!(entries[1].file, "bar.rs");+ assert_eq!(entries[1].desc, "second example");+ Ok(())+ }+}diff --git a/datafusion-examples/src/utils/example_metadata/render.rs b/datafusion-examples/src/utils/example_metadata/render.rsnew file mode 100644index 0000000000..1a4df3a400--- /dev/null+++ b/datafusion-examples/src/utils/example_metadata/render.rs@@ -0,0 +1,212 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Markdown renderer for DataFusion example documentation.+//!+//! This module takes parsed example metadata and generates the+//! `README.md` content for `datafusion-examples`, including group+//! sections and example tables.++use std::path::PathBuf;++use datafusion::error::{DataFusionError, Result};++use crate::utils::example_metadata::discover::discover_example_groups;+use crate::utils::example_metadata::model::ExampleGroup;+use crate::utils::example_metadata::{Category, RepoLayout};++const STATIC_HEADER: &str = r#"<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# DataFusion Examples++This crate includes end to end, highly commented examples of how to use+various DataFusion APIs to help you get started.++## Prerequisites++Run `git submodule update --init` to init test files.++## Running Examples++To run an example, use the `cargo run` command, such as:++```bash+git clone https://github.com/apache/datafusion+cd datafusion+# Download test data+git submodule update --init++# Change to the examples directory+cd datafusion-examples/examples++# Run all examples in a group+cargo run --example <group> -- all++# Run a specific example within a group+cargo run --example <group> -- <subcommand>++# Run all examples in the `dataframe` group+cargo run --example dataframe -- all++# Run a single example from the `dataframe` group+# (apply the same pattern for any other group)+cargo run --example dataframe -- dataframe+```+"#;++/// Well-known abbreviations used to preserve correct capitalization+/// when generating human-readable documentation titles.+pub const ABBREVIATIONS: &[(&str, &str)] = &[+ ("dataframe", "DataFrame"),+ ("io", "IO"),+ ("sql", "SQL"),+ ("udf", "UDF"),+];++/// Generates Markdown documentation for DataFusion examples.+///+/// If `group` is `None`, documentation is generated for all example groups.+/// If `group` is `Some`, only that group is rendered.+///+/// # Errors+///+/// Returns an error if:+/// - the requested group does not exist+/// - a `main.rs` file is missing+/// - documentation comments are malformed+pub fn generate_examples_readme(+ layout: &RepoLayout,+ group: Option<&str>,+) -> Result<String> {+ let examples_root = layout.examples_root();++ let mut out = String::new();+ out.push_str(STATIC_HEADER);++ let group_dirs: Vec<PathBuf> = match group {+ Some(name) => {+ let dir = examples_root.join(name);+ if !dir.is_dir() {+ return Err(DataFusionError::Execution(format!(+ "Example group `{name}` does not exist"+ )));+ }+ vec![dir]+ }+ None => discover_example_groups(&examples_root)?,+ };++ for group_dir in group_dirs {+ let raw_name =+ group_dir+ .file_name()+ .and_then(|s| s.to_str())+ .ok_or_else(|| {+ DataFusionError::Execution("Invalid example group dir".to_string())+ })?;++ let category = Category::for_group(raw_name);+ let group = ExampleGroup::from_dir(&group_dir, category)?;++ out.push_str(&group.render_markdown());+ }++ Ok(out)+}++impl ExampleGroup {+ /// Renders this example group as a Markdown section for the README.+ pub fn render_markdown(&self) -> String {+ let mut out = String::new();+ out.push_str(&format!("\n## {} Examples\n\n", self.name.title()));+ out.push_str(&format!("### Group: `{}`\n\n", self.name.raw()));+ out.push_str(&format!("#### Category: {}\n\n", self.category.name()));+ out.push_str("| Subcommand | File Path | Description |\n");+ out.push_str("| --- | --- | --- |\n");++ for example in &self.examples {+ out.push_str(&format!(+ "| {} | [`{}/{}`](examples/{}/{}) | {} |\n",+ example.subcommand,+ self.name.raw(),+ example.file,+ self.name.raw(),+ example.file,+ example.desc+ ));+ }++ out+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ use crate::utils::example_metadata::test_utils::assert_exec_err_contains;++ use std::fs;++ use tempfile::TempDir;++ #[test]+ fn single_group_generation_works() {+ let tmp = TempDir::new().unwrap();+ // Fake repo root+ let layout = RepoLayout::from_root(tmp.path().to_path_buf());++ // Create: datafusion-examples/examples/builtin_functions+ let examples_dir = layout.example_group_dir("builtin_functions");+ fs::create_dir_all(&examples_dir).unwrap();++ fs::write(+ examples_dir.join("main.rs"),+ "//! - `x`\n//! (file: foo.rs, desc: test)",+ )+ .unwrap();++ let out = generate_examples_readme(&layout, Some("builtin_functions")).unwrap();+ assert!(out.contains("Builtin Functions"));+ assert!(out.contains("| x | [`builtin_functions/foo.rs`]"));+ }++ #[test]+ fn single_group_generation_fails_if_group_missing() {+ let tmp = TempDir::new().unwrap();+ let layout = RepoLayout::from_root(tmp.path().to_path_buf());+ let err = generate_examples_readme(&layout, Some("missing_group")).unwrap_err();+ assert_exec_err_contains(err, "Example group `missing_group` does not exist");+ }+}diff --git a/datafusion-examples/src/utils/example_metadata/test_utils.rs b/datafusion-examples/src/utils/example_metadata/test_utils.rsnew file mode 100644index 0000000000..d6ab3b06ba--- /dev/null+++ b/datafusion-examples/src/utils/example_metadata/test_utils.rs@@ -0,0 +1,63 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Test helpers for example metadata parsing and validation.+//!+//! This module provides small, focused utilities to reduce duplication+//! and keep tests readable across the example metadata submodules.++use std::fs;++use datafusion::error::{DataFusionError, Result};+use tempfile::TempDir;++use crate::utils::example_metadata::{Category, ExampleGroup};++/// Asserts that an `Execution` error contains the expected message fragment.+///+/// Keeps tests focused on semantic error causes without coupling them+/// to full error string formatting.+pub fn assert_exec_err_contains(err: DataFusionError, needle: &str) {+ match err {+ DataFusionError::Execution(msg) => {+ assert!(+ msg.contains(needle),+ "expected '{needle}' in error message, got: {msg}"+ );+ }+ other => panic!("expected Execution error, got: {other:?}"),+ }+}++/// Helper for grammar-focused tests.+///+/// Creates a minimal temporary example group with a single `main.rs`+/// containing the provided docs. Intended for testing parsing and+/// validation rules, not full integration behavior.+pub fn example_group_from_docs(docs: &str) -> Result<ExampleGroup> {+ let tmp = TempDir::new().map_err(|e| {+ DataFusionError::Execution(format!("Failed initializing temp dir: {e}"))+ })?;+ let dir = tmp.path().join("group");+ fs::create_dir(&dir).map_err(|e| {+ DataFusionError::Execution(format!("Failed creating temp dir: {e}"))+ })?;+ fs::write(dir.join("main.rs"), docs).map_err(|e| {+ DataFusionError::Execution(format!("Failed writing to temp file: {e}"))+ })?;+ ExampleGroup::from_dir(&dir, Category::SingleProcess)+}diff --git a/datafusion-examples/src/utils/examples_docs.rs b/datafusion-examples/src/utils/examples_docs.rsdeleted file mode 100644index 4d8b58c736..0000000000--- a/datafusion-examples/src/utils/examples_docs.rs+++ /dev/null@@ -1,908 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--//! Documentation generator for DataFusion examples.-//!-//! # Design goals-//!-//! - Keep README.md in sync with runnable examples-//! - Fail fast on malformed documentation-//!-//! # Overview-//!-//! Each example group corresponds to a directory under-//! `datafusion-examples/examples/<group>` containing a `main.rs` file.-//! Documentation is extracted from structured `//!` comments in that file.-//!-//! For each example group, the generator produces:-//!-//! ```text-//! ## <Group Name> Examples-//! ### Group: `<group>`-//! #### Category: Single Process | Distributed-//!-//! | Subcommand | File Path | Description |-//! ```-//!-//! # Usage-//!-//! Generate documentation for a single group only:-//!-//! ```bash-//! cargo run --bin examples-docs -- dataframe-//! ```-//!-//! Generate documentation for all examples:-//!-//! ```bash-//! cargo run --bin examples-docs-//! ```--use std::collections::HashSet;-use std::fs;-use std::path::{Path, PathBuf};--use datafusion::error::{DataFusionError, Result};-use nom::{- IResult, Parser,- bytes::complete::{tag, take_until, take_while},- character::complete::multispace0,- combinator::all_consuming,- sequence::{delimited, preceded},-};--const STATIC_HEADER: &str = r#"<!---- Licensed to the Apache Software Foundation (ASF) under one- or more contributor license agreements. See the NOTICE file- distributed with this work for additional information- regarding copyright ownership. The ASF licenses this file- to you under the Apache License, Version 2.0 (the- "License"); you may not use this file except in compliance- with the License. You may obtain a copy of the License at-- http://www.apache.org/licenses/LICENSE-2.0-- Unless required by applicable law or agreed to in writing,- software distributed under the License is distributed on an- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY- KIND, either express or implied. See the License for the- specific language governing permissions and limitations- under the License.--->--# DataFusion Examples--This crate includes end to end, highly commented examples of how to use-various DataFusion APIs to help you get started.--## Prerequisites--Run `git submodule update --init` to init test files.--## Running Examples--To run an example, use the `cargo run` command, such as:--```bash-git clone https://github.com/apache/datafusion-cd datafusion-# Download test data-git submodule update --init--# Change to the examples directory-cd datafusion-examples/examples--# Run all examples in a group-cargo run --example <group> -- all--# Run a specific example within a group-cargo run --example <group> -- <subcommand>--# Run all examples in the `dataframe` group-cargo run --example dataframe -- all--# Run a single example from the `dataframe` group-# (apply the same pattern for any other group)-cargo run --example dataframe -- dataframe-```-"#;--/// Well-known abbreviations used to preserve correct capitalization-/// when generating human-readable documentation titles.-const ABBREVIATIONS: &[(&str, &str)] = &[- ("dataframe", "DataFrame"),- ("io", "IO"),- ("sql", "SQL"),- ("udf", "UDF"),-];--/// Describes the layout of a DataFusion repository.-///-/// This type centralizes knowledge about where example-related-/// directories live relative to the repository root.-#[derive(Debug, Clone)]-pub struct RepoLayout {- root: PathBuf,-}--impl From<&Path> for RepoLayout {- fn from(path: &Path) -> Self {- Self {- root: path.to_path_buf(),- }- }-}--impl RepoLayout {- /// Creates a layout from an explicit repository root.- pub fn from_root(root: PathBuf) -> Self {- Self { root }- }-- /// Detects the repository root based on `CARGO_MANIFEST_DIR`.- ///- /// This is intended for use from binaries inside the workspace.- pub fn detect() -> Result<Self> {- let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));-- let root = manifest_dir.parent().ok_or_else(|| {- DataFusionError::Execution(- "CARGO_MANIFEST_DIR does not have a parent".to_string(),- )- })?;-- Ok(Self {- root: root.to_path_buf(),- })- }-- /// Returns the repository root directory.- pub fn root(&self) -> &Path {- &self.root- }-- /// Returns the `datafusion-examples/examples` directory.- pub fn examples_root(&self) -> PathBuf {- self.root.join("datafusion-examples").join("examples")- }-- /// Returns the directory for a single example group.- ///- /// Example: `examples/udf`- pub fn example_group_dir(&self, group: &str) -> PathBuf {- self.examples_root().join(group)- }-}--/// A group of related examples (e.g. `builtin_functions`, `udf`).-///-/// Each group corresponds to a directory containing a `main.rs` file-/// with structured documentation comments.-#[derive(Debug)]-pub struct ExampleGroup {- pub name: GroupName,- pub examples: Vec<ExampleEntry>,- pub category: Category,-}--impl ExampleGroup {- /// Parses an example group from its directory.- ///- /// The group name is derived from the directory name, and example- /// entries are extracted from `main.rs`.- pub fn from_dir(dir: &Path, category: Category) -> Result<Self> {- let raw_name = dir- .file_name()- .and_then(|s| s.to_str())- .ok_or_else(|| {- DataFusionError::Execution("Invalid example group dir".to_string())- })?- .to_string();-- let name = GroupName::from_dir_name(raw_name);- let main_rs = dir.join("main.rs");- let examples = parse_main_rs_docs(&main_rs)?;-- Ok(Self {- name,- examples,- category,- })- }-- // Renders this example group as a Markdown section.- pub fn render_markdown(&self) -> String {- let mut out = String::new();- out.push_str(&format!("\n## {} Examples\n\n", self.name.title()));- out.push_str(&format!("### Group: `{}`\n\n", self.name.raw()));- out.push_str(&format!("#### Category: {}\n\n", self.category.name()));- out.push_str("| Subcommand | File Path | Description |\n");- out.push_str("| --- | --- | --- |\n");-- for ex in &self.examples {- out.push_str(&format!(- "| {} | [`{}/{}`](examples/{}/{}) | {} |\n",- ex.subcommand,- self.name.raw(),- ex.file,- self.name.raw(),- ex.file,- ex.desc- ));- }-- out- }-}--/// Represents an example group name in both raw and human-readable forms.-///-/// For example:-/// - raw: `builtin_functions`-/// - title: `Builtin Functions`-#[derive(Debug)]-pub struct GroupName {- raw: String,- title: String,-}--impl GroupName {- /// Creates a group name from a directory name.- pub fn from_dir_name(raw: String) -> Self {- let title = raw- .split('_')- .map(format_part)- .collect::<Vec<_>>()- .join(" ");-- Self { raw, title }- }-- /// Returns the raw group name (directory name).- pub fn raw(&self) -> &str {- &self.raw- }-- /// Returns a title-cased name for documentation.- pub fn title(&self) -> &str {- &self.title- }-}--/// A single runnable example within a group.-///-/// Each entry corresponds to a subcommand documented in `main.rs`.-#[derive(Debug)]-pub struct ExampleEntry {- /// CLI subcommand name.- pub subcommand: String,- /// Rust source file name.- pub file: String,- /// Human-readable description.- pub desc: String,-}--/// Execution category of an example group.-#[derive(Debug, Default)]-pub enum Category {- /// Runs in a single process.- #[default]- SingleProcess,- /// Requires a distributed setup.- Distributed,-}--impl Category {- /// Returns the display name used in documentation.- pub fn name(&self) -> &str {- match self {- Self::SingleProcess => "Single Process",- Self::Distributed => "Distributed",- }- }-- /// Determines the category for a group by name.- pub fn for_group(name: &str) -> Self {- match name {- "flight" => Category::Distributed,- _ => Category::SingleProcess,- }- }-}--/// Generates Markdown documentation for DataFusion examples.-///-/// If `group` is `None`, documentation is generated for all example groups.-/// If `group` is `Some`, only that group is rendered.-///-/// # Errors-///-/// Returns an error if:-/// - the requested group does not exist-/// - a `main.rs` file is missing-/// - documentation comments are malformed-pub fn generate_examples_readme(- layout: &RepoLayout,- group: Option<&str>,-) -> Result<String> {- let examples_root = layout.examples_root();-- let mut out = String::new();- out.push_str(STATIC_HEADER);-- let group_dirs: Vec<PathBuf> = match group {- Some(name) => {- let dir = examples_root.join(name);- if !dir.is_dir() {- return Err(DataFusionError::Execution(format!(- "Example group `{name}` does not exist"- )));- }- vec![dir]- }- None => discover_example_groups(&examples_root)?,- };-- for group_dir in group_dirs {- let raw_name =- group_dir- .file_name()- .and_then(|s| s.to_str())- .ok_or_else(|| {- DataFusionError::Execution("Invalid example group dir".to_string())- })?;-- let category = Category::for_group(raw_name);- let group = ExampleGroup::from_dir(&group_dir, category)?;-- out.push_str(&group.render_markdown());- }-- Ok(out)-}--/// Parsing state machine used while scanning `main.rs` docs.-///-/// This makes the "subcommand - metadata" relationship explicit:-/// metadata is only valid immediately after a subcommand has been seen.-enum ParserState<'a> {- /// Not currently expecting metadata.- Idle,- /// A subcommand was just parsed; the next valid metadata (if any)- /// must belong to this subcommand.- SeenSubcommand(&'a str),-}--/// Parses example entries from a group's `main.rs` file.-pub fn parse_main_rs_docs(path: &Path) -> Result<Vec<ExampleEntry>> {- let content = fs::read_to_string(path)?;- let mut entries = vec![];- let mut state = ParserState::Idle;- let mut seen_subcommands = HashSet::new();-- for (line_no, raw_line) in content.lines().enumerate() {- let line = raw_line.trim();-- // Try parsing subcommand, excluding `all` because it's not used in README- if let Ok((_, sub)) = parse_subcommand_line(line) {- state = if sub == "all" {- ParserState::Idle- } else {- ParserState::SeenSubcommand(sub)- };- continue;- }-- // Try parsing metadata- if let Ok((_, (file, desc))) = parse_metadata_line(line) {- let subcommand = match state {- ParserState::SeenSubcommand(s) => s,- ParserState::Idle => {- return Err(DataFusionError::Execution(format!(- "Metadata without preceding subcommand at {}:{}",- path.display(),- line_no + 1- )));- }- };-- if !seen_subcommands.insert(subcommand) {- return Err(DataFusionError::Execution(format!(- "Duplicate metadata for subcommand `{subcommand}`"- )));- }-- entries.push(ExampleEntry {- subcommand: subcommand.to_string(),- file: file.to_string(),- desc: desc.to_string(),- });-- state = ParserState::Idle;- continue;- }-- // If a non-blank doc line interrupts a pending subcommand, reset the state- if let ParserState::SeenSubcommand(_) = state- && is_non_blank_doc_line(line)- {- state = ParserState::Idle;- }- }-- Ok(entries)-}--/// Returns `true` for non-blank Rust doc comment lines (`//!`).-///-/// Used to detect when a subcommand is interrupted by unrelated documentation,-/// so metadata is only accepted immediately after a subcommand (blank doc lines-/// are allowed in between).-fn is_non_blank_doc_line(line: &str) -> bool {- line.starts_with("//!") && !line.trim_start_matches("//!").trim().is_empty()-}--/// Parses a subcommand declaration line from `main.rs` docs.-///-/// Expected format:-/// ```text-/// //! - `<subcommand>`-/// ```-fn parse_subcommand_line(input: &str) -> IResult<&str, &str> {- let parser = preceded(- multispace0,- delimited(tag("//! - `"), take_until("`"), tag("`")),- );- all_consuming(parser).parse(input)-}--/// Parses example metadata (file name and description) from `main.rs` docs.-///-/// Expected format:-/// ```text-/// //! (file: <file>.rs, desc: <description>)-/// ```-pub fn parse_metadata_line(input: &str) -> IResult<&str, (&str, &str)> {- let parser = preceded(- multispace0,- preceded(tag("//!"), preceded(multispace0, take_while(|_| true))),- );- let (rest, line) = all_consuming(parser).parse(input)?;-- let content = line- .strip_prefix("(")- .and_then(|s| s.strip_suffix(")"))- .ok_or_else(|| {- nom::Err::Error(nom::error::Error::new(line, nom::error::ErrorKind::Tag))- })?;-- let (file, desc) = content- .strip_prefix("file:")- .ok_or_else(|| {- nom::Err::Error(nom::error::Error::new(line, nom::error::ErrorKind::Tag))- })?- .split_once(", desc:")- .ok_or_else(|| {- nom::Err::Error(nom::error::Error::new(line, nom::error::ErrorKind::Tag))- })?;-- Ok((rest, (file.trim(), desc.trim())))-}--/// Discovers all example group directories under the given root.-fn discover_example_groups(root: &Path) -> Result<Vec<PathBuf>> {- let mut groups = Vec::new();- for entry in fs::read_dir(root)? {- let entry = entry?;- let path = entry.path();-- if path.is_dir() && path.join("main.rs").exists() {- groups.push(path);- }- }-- groups.sort();-- Ok(groups)-}--/// Formats a single group-name segment for display.-///-/// This function applies DataFusion-specific capitalization rules:-/// - Known abbreviations (e.g. `sql`, `io`, `udf`) are rendered in all caps-/// - All other segments fall back to standard Title Case-fn format_part(part: &str) -> String {- let lower = part.to_ascii_lowercase();-- if let Some((_, replacement)) = ABBREVIATIONS.iter().find(|(k, _)| *k == lower) {- return replacement.to_string();- }-- let mut chars = part.chars();- match chars.next() {- Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),- None => String::new(),- }-}--#[cfg(test)]-mod tests {- use super::*;-- use tempfile::TempDir;-- /// Helper for grammar-focused tests.- ///- /// Creates a minimal temporary example group with a single `main.rs`- /// containing the provided docs. Intended for testing parsing and- /// validation rules, not full integration behavior.- fn example_group_from_docs(docs: &str) -> Result<ExampleGroup> {- let tmp = TempDir::new().map_err(|e| {- DataFusionError::Execution(format!("Failed initializing temp dir: {e}"))- })?;- let dir = tmp.path().join("group");- fs::create_dir(&dir).map_err(|e| {- DataFusionError::Execution(format!("Failed creating temp dir: {e}"))- })?;- fs::write(dir.join("main.rs"), docs).map_err(|e| {- DataFusionError::Execution(format!("Failed writing to temp file: {e}"))- })?;- ExampleGroup::from_dir(&dir, Category::SingleProcess)- }-- /// Asserts that an `Execution` error contains the expected message fragment.- ///- /// Keeps tests focused on semantic error causes without coupling them- /// to full error string formatting.- fn assert_exec_err_contains(err: DataFusionError, needle: &str) {- match err {- DataFusionError::Execution(msg) => {- assert!(- msg.contains(needle),- "expected '{needle}' in error message, got: {msg}"- );- }- other => panic!("expected Execution error, got: {other:?}"),- }- }-- #[test]- fn all_subcommand_is_ignored() -> Result<()> {- let group = example_group_from_docs(- r#"- //! - `all` — run all examples included in this module- //!- //! - `foo`- //! (file: foo.rs, desc: foo example)- "#,- )?;- assert_eq!(group.examples.len(), 1);- assert_eq!(group.examples[0].subcommand, "foo");- Ok(())- }-- #[test]- fn parse_subcommand_line_accepts_valid_input() {- let line = "//! - `date_time`";- let sub = parse_subcommand_line(line);- assert_eq!(sub, Ok(("", "date_time")));- }-- #[test]- fn parse_subcommand_line_invalid_inputs() {- let err_lines = [- "//! - ",- "//! - foo",- "//! - `foo` bar",- "//! --",- "//!-",- "//!--",- "//!",- "//",- "/",- "",- ];- for line in err_lines {- assert!(- parse_subcommand_line(line).is_err(),- "expected error for input: {line}"- );- }- }-- #[test]- fn parse_metadata_line_accepts_valid_input() {- let line =- "//! (file: date_time.rs, desc: Examples of date-time related functions)";- let res = parse_metadata_line(line);- assert_eq!(- res,- Ok((- "",- ("date_time.rs", "Examples of date-time related functions")- ))- );-- let line = "//! (file: foo.rs, desc: Foo, bar, baz)";- let res = parse_metadata_line(line);- assert_eq!(res, Ok(("", ("foo.rs", "Foo, bar, baz"))));-- let line = "//! (file: foo.rs, desc: Foo(FOO))";- let res = parse_metadata_line(line);- assert_eq!(res, Ok(("", ("foo.rs", "Foo(FOO)"))));- }-- #[test]- fn parse_metadata_line_invalid_inputs() {- let bad_lines = [- "//! (file: foo.rs)",- "//! (desc: missing file)",- "//! file: foo.rs, desc: test",- "//! file: foo.rs,desc: test",- "//! (file: foo.rs desc: test)",- "//! (file: foo.rs,desc: test)",- "//! (desc: test, file: foo.rs)",- "//! ()",- "//! (file: foo.rs, desc: test) extra",- "",- ];- for line in bad_lines {- assert!(- parse_metadata_line(line).is_err(),- "expected error for input: {line}"- );- }- }-- #[test]- fn metadata_without_subcommand_fails() {- let err = example_group_from_docs("//! (file: foo.rs, desc: missing subcommand)")- .unwrap_err();- assert_exec_err_contains(err, "Metadata without preceding subcommand");- }-- #[test]- fn parse_group_example_works() -> Result<()> {- let tmp = TempDir::new().unwrap();-- // Simulate: examples/builtin_functions/- let group_dir = tmp.path().join("builtin_functions");- fs::create_dir(&group_dir)?;-- // Write a fake main.rs with docs- let main_rs = group_dir.join("main.rs");- fs::write(- &main_rs,- r#"- // Licensed to the Apache Software Foundation (ASF) under one- // or more contributor license agreements. See the NOTICE file- // distributed with this work for additional information- // regarding copyright ownership. The ASF licenses this file- // to you under the Apache License, Version 2.0 (the- // "License"); you may not use this file except in compliance- // with the License. You may obtain a copy of the License at- //- // http://www.apache.org/licenses/LICENSE-2.0- //- // Unless required by applicable law or agreed to in writing,- // software distributed under the License is distributed on an- // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY- // KIND, either express or implied. See the License for the- // specific language governing permissions and limitations- // under the License.- //- //! # These are miscellaneous function-related examples- //!- //! These examples demonstrate miscellaneous function-related features.- //!- //! ## Usage- //! ```bash- //! cargo run --example builtin_functions -- [all|date_time|function_factory|regexp]- //! ```- //!- //! Each subcommand runs a corresponding example:- //! - `all` — run all examples included in this module- //!- //! - `date_time`- //! (file: date_time.rs, desc: Examples of date-time related functions and queries)- //!- //! - `function_factory`- //! (file: function_factory.rs, desc: Register `CREATE FUNCTION` handler to implement SQL macros)- //!- //! - `regexp`- //! (file: regexp.rs, desc: Examples of using regular expression functions)- "#,- )?;-- let group = ExampleGroup::from_dir(&group_dir, Category::SingleProcess)?;-- // Assert group-level data- assert_eq!(group.name.title(), "Builtin Functions");- assert_eq!(group.examples.len(), 3);-- // Assert 1 example- assert_eq!(group.examples[0].subcommand, "date_time");- assert_eq!(group.examples[0].file, "date_time.rs");- assert_eq!(- group.examples[0].desc,- "Examples of date-time related functions and queries"- );-- // Assert 2 example- assert_eq!(group.examples[1].subcommand, "function_factory");- assert_eq!(group.examples[1].file, "function_factory.rs");- assert_eq!(- group.examples[1].desc,- "Register `CREATE FUNCTION` handler to implement SQL macros"- );-- // Assert 3 example- assert_eq!(group.examples[2].subcommand, "regexp");- assert_eq!(group.examples[2].file, "regexp.rs");- assert_eq!(- group.examples[2].desc,- "Examples of using regular expression functions"- );-- Ok(())- }-- #[test]- fn single_group_generation_works() {- let tmp = TempDir::new().unwrap();- // Fake repo root- let layout = RepoLayout::from_root(tmp.path().to_path_buf());-- // Create: datafusion-examples/examples/builtin_functions- let examples_dir = layout.example_group_dir("builtin_functions");- fs::create_dir_all(&examples_dir).unwrap();-- fs::write(- examples_dir.join("main.rs"),- "//! - `x`\n//! (file: foo.rs, desc: test)",- )- .unwrap();-- let out = generate_examples_readme(&layout, Some("builtin_functions")).unwrap();- assert!(out.contains("Builtin Functions"));- }-- #[test]- fn single_group_generation_fails_if_group_missing() {- let tmp = TempDir::new().unwrap();- let layout = RepoLayout::from_root(tmp.path().to_path_buf());- let err = generate_examples_readme(&layout, Some("missing_group")).unwrap_err();- assert_exec_err_contains(err, "Example group `missing_group` does not exist");- }-- #[test]- fn group_name_handles_abbreviations() {- assert_eq!(- GroupName::from_dir_name("dataframe".to_string()).title(),- "DataFrame"- );- assert_eq!(- GroupName::from_dir_name("data_io".to_string()).title(),- "Data IO"- );- assert_eq!(- GroupName::from_dir_name("sql_ops".to_string()).title(),- "SQL Ops"- );- assert_eq!(GroupName::from_dir_name("udf".to_string()).title(), "UDF");- }-- #[test]- fn duplicate_metadata_without_repeating_subcommand_fails() {- let err = example_group_from_docs(- r#"- //! - `foo`- //! (file: a.rs, desc: first)- //! (file: b.rs, desc: second)- "#,- )- .unwrap_err();- assert_exec_err_contains(err, "Metadata without preceding subcommand");- }-- #[test]- fn duplicate_metadata_for_same_subcommand_fails() {- let err = example_group_from_docs(- r#"- //! - `foo`- //! (file: a.rs, desc: first)- //!- //! - `foo`- //! (file: b.rs, desc: second)- "#,- )- .unwrap_err();- assert_exec_err_contains(err, "Duplicate metadata for subcommand `foo`");- }-- #[test]- fn metadata_must_follow_subcommand() {- let err = example_group_from_docs(- r#"- //! - `foo`- //! some unrelated comment- //! (file: foo.rs, desc: test)- "#,- )- .unwrap_err();- assert_exec_err_contains(err, "Metadata without preceding subcommand");- }-- #[test]- fn preserves_example_order_from_main_rs() -> Result<()> {- let group = example_group_from_docs(- r#"- //! - `second`- //! (file: second.rs, desc: second example)- //!- //! - `first`- //! (file: first.rs, desc: first example)- //!- //! - `third`- //! (file: third.rs, desc: third example)- "#,- )?;-- let subcommands: Vec<&str> = group- .examples- .iter()- .map(|e| e.subcommand.as_str())- .collect();-- assert_eq!(- subcommands,- vec!["second", "first", "third"],- "examples must preserve the order defined in main.rs"- );-- Ok(())- }-- #[test]- fn metadata_can_follow_blank_doc_line() -> Result<()> {- let group = example_group_from_docs(- r#"- //! - `foo`- //!- //! (file: foo.rs, desc: test)- "#,- )?;- assert_eq!(group.examples.len(), 1);- Ok(())- }-- #[test]- fn group_name_title_cases() {- let cases = [- ("very_long_group_name", "Very Long Group Name"),- ("foo", "Foo"),- ("dataframe", "DataFrame"),- ("data_io", "Data IO"),- ("sql_ops", "SQL Ops"),- ("udf", "UDF"),- ];- for (input, expected) in cases {- let name = GroupName::from_dir_name(input.to_string());- assert_eq!(name.title(), expected);- }- }-}diff --git a/datafusion-examples/src/utils/mod.rs b/datafusion-examples/src/utils/mod.rsindex aa44491296..da96724a49 100644--- a/datafusion-examples/src/utils/mod.rs+++ b/datafusion-examples/src/utils/mod.rs@@ -17,6 +17,6 @@mod csv_to_parquet;pub mod datasets;-pub mod examples_docs;+pub mod example_metadata;pub use csv_to_parquet::write_csv_to_parquet;