Apache DataFusion SQL Query Engine
Cleanup example metadata parsing utilities(#20251) (#20252)
Sergey Zhukov17416bf92f5b
diff --git a/datafusion-examples/src/utils/csv_to_parquet.rs b/datafusion-examples/src/utils/csv_to_parquet.rsindex 16541b13ae..1fbf2930e9 100644--- a/datafusion-examples/src/utils/csv_to_parquet.rs+++ b/datafusion-examples/src/utils/csv_to_parquet.rs@@ -18,9 +18,8 @@use std::path::{Path, PathBuf};use datafusion::dataframe::DataFrameWriteOptions;-use datafusion::error::Result;+use datafusion::error::{DataFusionError, Result};use datafusion::prelude::{CsvReadOptions, SessionContext};-use datafusion_common::DataFusionError;use tempfile::TempDir;use tokio::fs::create_dir_all;diff --git a/datafusion-examples/src/utils/datasets/mod.rs b/datafusion-examples/src/utils/datasets/mod.rsindex 47f946f7d8..1857e6af9b 100644--- a/datafusion-examples/src/utils/datasets/mod.rs+++ b/datafusion-examples/src/utils/datasets/mod.rs@@ -18,8 +18,7 @@use std::path::PathBuf;use arrow_schema::SchemaRef;-use datafusion::error::Result;-use datafusion_common::DataFusionError;+use datafusion::error::{DataFusionError, Result};pub mod cars;pub mod regex;@@ -50,10 +49,11 @@ impl ExampleDataset {}pub fn path_str(&self) -> Result<String> {- self.path().to_str().map(String::from).ok_or_else(|| {+ let path = self.path();+ path.to_str().map(String::from).ok_or_else(|| {DataFusionError::Execution(format!("CSV directory path is not valid UTF-8: {}",- self.path().display()+ path.display()))})}diff --git a/datafusion-examples/src/utils/example_metadata/discover.rs b/datafusion-examples/src/utils/example_metadata/discover.rsindex c5cf3ec111..1ba5f6d29a 100644--- a/datafusion-examples/src/utils/example_metadata/discover.rs+++ b/datafusion-examples/src/utils/example_metadata/discover.rs@@ -20,10 +20,12 @@//! An example group is defined as a directory containing a `main.rs` file//! under the examples root. This module is intentionally filesystem-focused//! and does not perform any parsing or rendering.+//! Discovery fails if no valid example groups are found.use std::fs;use std::path::{Path, PathBuf};+use datafusion::common::exec_err;use datafusion::error::Result;/// Discovers all example group directories under the given root.@@ -35,10 +37,15 @@ pub fn discover_example_groups(root: &Path) -> Result<Vec<PathBuf>> {let entry = entry?;let path = entry.path();- if path.is_dir() && path.join("main.rs").exists() {+ if path.is_dir() && path.join("main.rs").is_file() {groups.push(path);}}++ if groups.is_empty() {+ return exec_err!("No example groups found under: {}", root.display());+ }+groups.sort();Ok(groups)}@@ -47,6 +54,8 @@ pub fn discover_example_groups(root: &Path) -> Result<Vec<PathBuf>> {mod tests {use super::*;+ use crate::utils::example_metadata::test_utils::assert_exec_err_contains;+use std::fs::{self, File};use tempfile::TempDir;@@ -66,10 +75,29 @@ mod tests {fs::create_dir(&group2)?;let groups = discover_example_groups(root)?;-assert_eq!(groups.len(), 1);assert_eq!(groups[0], group1);+ Ok(())+ }++ #[test]+ fn discover_example_groups_errors_if_main_rs_is_a_directory() -> Result<()> {+ let tmp = TempDir::new()?;+ let root = tmp.path();+ let group = root.join("group");+ fs::create_dir(&group)?;+ fs::create_dir(group.join("main.rs"))?;++ let err = discover_example_groups(root).unwrap_err();+ assert_exec_err_contains(err, "No example groups found");+ Ok(())+ }+ #[test]+ fn discover_example_groups_errors_if_none_found() -> Result<()> {+ let tmp = TempDir::new()?;+ let err = discover_example_groups(tmp.path()).unwrap_err();+ assert_exec_err_contains(err, "No example groups found");Ok(())}}diff --git a/datafusion-examples/src/utils/example_metadata/model.rs b/datafusion-examples/src/utils/example_metadata/model.rsindex 74b605cde1..11416d141e 100644--- a/datafusion-examples/src/utils/example_metadata/model.rs+++ b/datafusion-examples/src/utils/example_metadata/model.rs@@ -25,7 +25,16 @@ use std::path::Path;use datafusion::error::{DataFusionError, Result};-use crate::utils::example_metadata::{parse_main_rs_docs, render::ABBREVIATIONS};+use crate::utils::example_metadata::parse_main_rs_docs;++/// Well-known abbreviations used to preserve correct capitalization+/// when generating human-readable documentation titles.+const ABBREVIATIONS: &[(&str, &str)] = &[+ ("dataframe", "DataFrame"),+ ("io", "IO"),+ ("sql", "SQL"),+ ("udf", "UDF"),+];/// A group of related examples (e.g. `builtin_functions`, `udf`).///diff --git a/datafusion-examples/src/utils/example_metadata/parser.rs b/datafusion-examples/src/utils/example_metadata/parser.rsindex 83105e7d40..4ead3e5a2a 100644--- a/datafusion-examples/src/utils/example_metadata/parser.rs+++ b/datafusion-examples/src/utils/example_metadata/parser.rs@@ -21,15 +21,16 @@//! and their associated metadata (file name and description), enforcing//! a strict ordering and structure to avoid ambiguous documentation.-use std::path::Path;-use std::{collections::HashSet, fs};+use std::{collections::HashSet, fs, path::Path};-use datafusion_common::{DataFusionError, Result};+use datafusion::common::exec_err;+use datafusion::error::Result;use nom::{- IResult, Parser,+ Err, IResult, Parser,bytes::complete::{tag, take_until, take_while},character::complete::multispace0,combinator::all_consuming,+ error::{Error, ErrorKind},sequence::{delimited, preceded},};@@ -77,19 +78,13 @@ fn parse_metadata_line(input: &str) -> IResult<&str, (&str, &str)> {let content = payload.strip_prefix("(").and_then(|s| s.strip_suffix(")"))- .ok_or_else(|| {- nom::Err::Error(nom::error::Error::new(payload, nom::error::ErrorKind::Tag))- })?;+ .ok_or_else(|| Err::Error(Error::new(payload, ErrorKind::Tag)))?;let (file, desc) = content.strip_prefix("file:")- .ok_or_else(|| {- nom::Err::Error(nom::error::Error::new(payload, nom::error::ErrorKind::Tag))- })?+ .ok_or_else(|| Err::Error(Error::new(payload, ErrorKind::Tag)))?.split_once(", desc:")- .ok_or_else(|| {- nom::Err::Error(nom::error::Error::new(payload, nom::error::ErrorKind::Tag))- })?;+ .ok_or_else(|| Err::Error(Error::new(payload, ErrorKind::Tag)))?;Ok((rest, (file.trim(), desc.trim())))}@@ -119,18 +114,16 @@ pub fn parse_main_rs_docs(path: &Path) -> Result<Vec<ExampleEntry>> {let subcommand = match state {ParserState::SeenSubcommand(s) => s,ParserState::Idle => {- return Err(DataFusionError::Execution(format!(+ return exec_err!("Metadata without preceding subcommand at {}:{}",path.display(),line_no + 1- )));+ );}};if !seen_subcommands.insert(subcommand) {- return Err(DataFusionError::Execution(format!(- "Duplicate metadata for subcommand `{subcommand}`"- )));+ return exec_err!("Duplicate metadata for subcommand `{subcommand}`");}entries.push(ExampleEntry {diff --git a/datafusion-examples/src/utils/example_metadata/render.rs b/datafusion-examples/src/utils/example_metadata/render.rsindex 1a4df3a400..a4ea620e78 100644--- a/datafusion-examples/src/utils/example_metadata/render.rs+++ b/datafusion-examples/src/utils/example_metadata/render.rs@@ -85,15 +85,6 @@ cargo run --example dataframe -- dataframe```"#;-/// Well-known abbreviations used to preserve correct capitalization-/// when generating human-readable documentation titles.-pub const ABBREVIATIONS: &[(&str, &str)] = &[- ("dataframe", "DataFrame"),- ("io", "IO"),- ("sql", "SQL"),- ("udf", "UDF"),-];-/// Generates Markdown documentation for DataFusion examples.////// If `group` is `None`, documentation is generated for all example groups.