Apache DataFusion SQL Query Engine
examples: replace sql_dialect with custom_sql_parser example (#19383)
Geoffrey Claudeef2c1a30eb76
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 1469aad541..8f38b38990 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -191,12 +191,12 @@ cargo run --example dataframe -- dataframe#### Category: Single Process-| Subcommand | File Path | Description |-| ---------- | ----------------------------------------------------- | ------------------------------ |-| analysis | [`sql_ops/analysis.rs`](examples/sql_ops/analysis.rs) | Analyze SQL queries |-| dialect | [`sql_ops/dialect.rs`](examples/sql_ops/dialect.rs) | Implement a custom SQL dialect |-| frontend | [`sql_ops/frontend.rs`](examples/sql_ops/frontend.rs) | Build LogicalPlans from SQL |-| query | [`sql_ops/query.rs`](examples/sql_ops/query.rs) | Query data using SQL |+| Subcommand | File Path | Description |+| ----------------- | ----------------------------------------------------------------------- | -------------------------------------------------- |+| analysis | [`sql_ops/analysis.rs`](examples/sql_ops/analysis.rs) | Analyze SQL queries |+| custom_sql_parser | [`sql_ops/custom_sql_parser.rs`](examples/sql_ops/custom_sql_parser.rs) | Implement a custom SQL parser to extend DataFusion |+| frontend | [`sql_ops/frontend.rs`](examples/sql_ops/frontend.rs) | Build LogicalPlans from SQL |+| query | [`sql_ops/query.rs`](examples/sql_ops/query.rs) | Query data using SQL |## UDF Examplesdiff --git a/datafusion-examples/examples/sql_ops/custom_sql_parser.rs b/datafusion-examples/examples/sql_ops/custom_sql_parser.rsnew file mode 100644index 0000000000..308a0de62a--- /dev/null+++ b/datafusion-examples/examples/sql_ops/custom_sql_parser.rs@@ -0,0 +1,420 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! This example demonstrates extending the DataFusion SQL parser to support+//! custom DDL statements, specifically `CREATE EXTERNAL CATALOG`.+//!+//! ### Custom Syntax+//! ```sql+//! CREATE EXTERNAL CATALOG my_catalog+//! STORED AS ICEBERG+//! LOCATION 's3://my-bucket/warehouse/'+//! OPTIONS (+//! 'region' = 'us-west-2'+//! );+//! ```+//!+//! Note: For the purpose of this example, we use `local://workspace/` to+//! automatically discover and register files from the project's test data.++use std::collections::HashMap;+use std::fmt::Display;+use std::sync::Arc;++use datafusion::catalog::{+ CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider,+ TableProviderFactory,+};+use datafusion::datasource::listing_table_factory::ListingTableFactory;+use datafusion::error::{DataFusionError, Result};+use datafusion::prelude::SessionContext;+use datafusion::sql::{+ parser::{DFParser, DFParserBuilder, Statement},+ sqlparser::{+ ast::{ObjectName, Value},+ keywords::Keyword,+ tokenizer::Token,+ },+};+use datafusion_common::{DFSchema, TableReference, plan_datafusion_err, plan_err};+use datafusion_expr::CreateExternalTable;+use futures::StreamExt;+use insta::assert_snapshot;+use object_store::ObjectStore;+use object_store::local::LocalFileSystem;++/// Entry point for the example.+pub async fn custom_sql_parser() -> Result<()> {+ // Use standard Parquet testing data as our "external" source.+ let base_path = datafusion::common::test_util::parquet_test_data();+ let base_path = std::path::Path::new(&base_path).canonicalize()?;++ // Make the path relative to the workspace root+ let workspace_root = workspace_root();+ let location = base_path+ .strip_prefix(&workspace_root)+ .map(|p| p.to_string_lossy().to_string())+ .unwrap_or_else(|_| base_path.to_string_lossy().to_string());++ let create_catalog_sql = format!(+ "CREATE EXTERNAL CATALOG parquet_testing+ STORED AS parquet+ LOCATION 'local://workspace/{location}'+ OPTIONS (+ 'schema_name' = 'staged_data',+ 'format.pruning' = 'true'+ )"+ );++ // =========================================================================+ // Part 1: Standard DataFusion parser rejects the custom DDL+ // =========================================================================+ println!("=== Part 1: Standard DataFusion Parser ===\n");+ println!("Parsing: {}\n", create_catalog_sql.trim());++ let ctx_standard = SessionContext::new();+ let err = ctx_standard+ .sql(&create_catalog_sql)+ .await+ .expect_err("Expected the standard parser to reject CREATE EXTERNAL CATALOG (custom DDL syntax)");++ println!("Error: {err}\n");+ assert_snapshot!(err.to_string(), @r#"SQL error: ParserError("Expected: TABLE, found: CATALOG at Line: 1, Column: 17")"#);++ // =========================================================================+ // Part 2: Custom parser handles the statement+ // =========================================================================+ println!("=== Part 2: Custom Parser ===\n");+ println!("Parsing: {}\n", create_catalog_sql.trim());++ let ctx = SessionContext::new();++ let mut parser = CustomParser::new(&create_catalog_sql)?;+ let statement = parser.parse_statement()?;+ match statement {+ CustomStatement::CreateExternalCatalog(stmt) => {+ handle_create_external_catalog(&ctx, stmt).await?;+ }+ CustomStatement::DFStatement(_) => {+ panic!("Expected CreateExternalCatalog statement");+ }+ }++ // Query a table from the registered catalog+ let query_sql = "SELECT id, bool_col, tinyint_col FROM parquet_testing.staged_data.alltypes_plain LIMIT 5";+ println!("Executing: {query_sql}\n");++ let results = execute_sql(&ctx, query_sql).await?;+ println!("{results}");+ assert_snapshot!(results, @r"+ +----+----------+-------------++ | id | bool_col | tinyint_col |+ +----+----------+-------------++ | 4 | true | 0 |+ | 5 | false | 1 |+ | 6 | true | 0 |+ | 7 | false | 1 |+ | 2 | true | 0 |+ +----+----------+-------------++ ");++ Ok(())+}++/// Execute SQL and return formatted results.+async fn execute_sql(ctx: &SessionContext, sql: &str) -> Result<String> {+ let batches = ctx.sql(sql).await?.collect().await?;+ Ok(arrow::util::pretty::pretty_format_batches(&batches)?.to_string())+}++/// Custom handler for the `CREATE EXTERNAL CATALOG` statement.+async fn handle_create_external_catalog(+ ctx: &SessionContext,+ stmt: CreateExternalCatalog,+) -> Result<()> {+ let factory = ListingTableFactory::new();+ let catalog = Arc::new(MemoryCatalogProvider::new());+ let schema = Arc::new(MemorySchemaProvider::new());++ // Extract options+ let mut schema_name = "public".to_string();+ let mut table_options = HashMap::new();++ for (k, v) in stmt.options {+ let val_str = match v {+ Value::SingleQuotedString(ref s) | Value::DoubleQuotedString(ref s) => {+ s.to_string()+ }+ Value::Number(ref n, _) => n.to_string(),+ Value::Boolean(b) => b.to_string(),+ _ => v.to_string(),+ };++ if k == "schema_name" {+ schema_name = val_str;+ } else {+ table_options.insert(k, val_str);+ }+ }++ println!(" Target Catalog: {}", stmt.name);+ println!(" Data Location: {}", stmt.location);+ println!(" Resolved Schema: {schema_name}");++ // Register a local object store rooted at the workspace root.+ // We use a specific authority 'workspace' to ensure consistent resolution.+ let store = Arc::new(LocalFileSystem::new_with_prefix(workspace_root())?);+ let store_url = url::Url::parse("local://workspace").unwrap();+ ctx.register_object_store(&store_url, Arc::clone(&store) as _);++ let target_ext = format!(".{}", stmt.catalog_type.to_lowercase());++ // For 'local://workspace/parquet-testing/data', the path is 'parquet-testing/data'.+ let path_str = stmt+ .location+ .strip_prefix("local://workspace/")+ .unwrap_or(&stmt.location);+ let prefix = object_store::path::Path::from(path_str);++ // Discover data files using the ObjectStore API+ let mut table_count = 0;+ let mut list_stream = store.list(Some(&prefix));++ while let Some(meta) = list_stream.next().await {+ let meta = meta?;+ let path = &meta.location;++ if path.as_ref().ends_with(&target_ext) {+ let name = std::path::Path::new(path.as_ref())+ .file_stem()+ .unwrap()+ .to_string_lossy()+ .to_string();++ let table_url = format!("local://workspace/{path}");++ let cmd = CreateExternalTable::builder(+ TableReference::bare(name.clone()),+ table_url,+ stmt.catalog_type.clone(),+ Arc::new(DFSchema::empty()),+ )+ .with_options(table_options.clone())+ .build();++ match factory.create(&ctx.state(), &cmd).await {+ Ok(table) => {+ schema.register_table(name, table)?;+ table_count += 1;+ }+ Err(e) => {+ eprintln!("Failed to create table {name}: {e}");+ }+ }+ }+ }+ println!(" Registered {table_count} tables into schema: {schema_name}");++ catalog.register_schema(&schema_name, schema)?;+ ctx.register_catalog(stmt.name.to_string(), catalog);++ Ok(())+}++/// Possible statements returned by our custom parser.+#[derive(Debug, Clone)]+pub enum CustomStatement {+ /// Standard DataFusion statement+ DFStatement(Box<Statement>),+ /// Custom `CREATE EXTERNAL CATALOG` statement+ CreateExternalCatalog(CreateExternalCatalog),+}++/// Data structure for `CREATE EXTERNAL CATALOG`.+#[derive(Debug, Clone)]+pub struct CreateExternalCatalog {+ pub name: ObjectName,+ pub catalog_type: String,+ pub location: String,+ pub options: Vec<(String, Value)>,+}++impl Display for CustomStatement {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::DFStatement(s) => write!(f, "{s}"),+ Self::CreateExternalCatalog(s) => write!(f, "{s}"),+ }+ }+}++impl Display for CreateExternalCatalog {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(+ f,+ "CREATE EXTERNAL CATALOG {} STORED AS {} LOCATION '{}'",+ self.name, self.catalog_type, self.location+ )?;+ if !self.options.is_empty() {+ write!(f, " OPTIONS (")?;+ for (i, (k, v)) in self.options.iter().enumerate() {+ if i > 0 {+ write!(f, ", ")?;+ }+ write!(f, "'{k}' = '{v}'")?;+ }+ write!(f, ")")?;+ }+ Ok(())+ }+}++/// A parser that extends `DFParser` with custom syntax.+struct CustomParser<'a> {+ df_parser: DFParser<'a>,+}++impl<'a> CustomParser<'a> {+ fn new(sql: &'a str) -> Result<Self> {+ Ok(Self {+ df_parser: DFParserBuilder::new(sql).build()?,+ })+ }++ pub fn parse_statement(&mut self) -> Result<CustomStatement> {+ if self.is_create_external_catalog() {+ return self.parse_create_external_catalog();+ }+ Ok(CustomStatement::DFStatement(Box::new(+ self.df_parser.parse_statement()?,+ )))+ }++ fn is_create_external_catalog(&self) -> bool {+ let t1 = &self.df_parser.parser.peek_nth_token(0).token;+ let t2 = &self.df_parser.parser.peek_nth_token(1).token;+ let t3 = &self.df_parser.parser.peek_nth_token(2).token;++ matches!(t1, Token::Word(w) if w.keyword == Keyword::CREATE)+ && matches!(t2, Token::Word(w) if w.keyword == Keyword::EXTERNAL)+ && matches!(t3, Token::Word(w) if w.value.to_uppercase() == "CATALOG")+ }++ fn parse_create_external_catalog(&mut self) -> Result<CustomStatement> {+ // Consume prefix tokens: CREATE EXTERNAL CATALOG+ for _ in 0..3 {+ self.df_parser.parser.next_token();+ }++ let name = self+ .df_parser+ .parser+ .parse_object_name(false)+ .map_err(|e| DataFusionError::External(Box::new(e)))?;++ let mut catalog_type = None;+ let mut location = None;+ let mut options = vec![];++ while let Some(keyword) = self.df_parser.parser.parse_one_of_keywords(&[+ Keyword::STORED,+ Keyword::LOCATION,+ Keyword::OPTIONS,+ ]) {+ match keyword {+ Keyword::STORED => {+ if catalog_type.is_some() {+ return plan_err!("Duplicate STORED AS");+ }+ self.df_parser+ .parser+ .expect_keyword(Keyword::AS)+ .map_err(|e| DataFusionError::External(Box::new(e)))?;+ catalog_type = Some(+ self.df_parser+ .parser+ .parse_identifier()+ .map_err(|e| DataFusionError::External(Box::new(e)))?+ .value,+ );+ }+ Keyword::LOCATION => {+ if location.is_some() {+ return plan_err!("Duplicate LOCATION");+ }+ location = Some(+ self.df_parser+ .parser+ .parse_literal_string()+ .map_err(|e| DataFusionError::External(Box::new(e)))?,+ );+ }+ Keyword::OPTIONS => {+ if !options.is_empty() {+ return plan_err!("Duplicate OPTIONS");+ }+ options = self.parse_value_options()?;+ }+ _ => unreachable!(),+ }+ }++ Ok(CustomStatement::CreateExternalCatalog(+ CreateExternalCatalog {+ name,+ catalog_type: catalog_type+ .ok_or_else(|| plan_datafusion_err!("Missing STORED AS"))?,+ location: location+ .ok_or_else(|| plan_datafusion_err!("Missing LOCATION"))?,+ options,+ },+ ))+ }++ /// Parse options in the form: (key [=] value, key [=] value, ...)+ fn parse_value_options(&mut self) -> Result<Vec<(String, Value)>> {+ let mut options = vec![];+ self.df_parser+ .parser+ .expect_token(&Token::LParen)+ .map_err(|e| DataFusionError::External(Box::new(e)))?;++ loop {+ let key = self.df_parser.parse_option_key()?;+ // Support optional '=' between key and value+ let _ = self.df_parser.parser.consume_token(&Token::Eq);+ let value = self.df_parser.parse_option_value()?;+ options.push((key, value));++ let comma = self.df_parser.parser.consume_token(&Token::Comma);+ if self.df_parser.parser.consume_token(&Token::RParen) {+ break;+ } else if !comma {+ return plan_err!("Expected ',' or ')' in OPTIONS");+ }+ }+ Ok(options)+ }+}++/// Returns the workspace root directory (parent of datafusion-examples).+fn workspace_root() -> std::path::PathBuf {+ std::path::Path::new(env!("CARGO_MANIFEST_DIR"))+ .parent()+ .expect("CARGO_MANIFEST_DIR should have a parent")+ .to_path_buf()+}diff --git a/datafusion-examples/examples/sql_ops/dialect.rs b/datafusion-examples/examples/sql_ops/dialect.rsdeleted file mode 100644index 62a9f3fa56..0000000000--- a/datafusion-examples/examples/sql_ops/dialect.rs+++ /dev/null@@ -1,135 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--//! See `main.rs` for how to run it.--use std::fmt::Display;--use datafusion::error::{DataFusionError, Result};-use datafusion::sql::{- parser::{CopyToSource, CopyToStatement, DFParser, DFParserBuilder, Statement},- sqlparser::{keywords::Keyword, tokenizer::Token},-};--/// This example demonstrates how to use the DFParser to parse a statement in a custom way-///-/// This technique can be used to implement a custom SQL dialect, for example.-pub async fn dialect() -> Result<()> {- let mut my_parser =- MyParser::new("COPY source_table TO 'file.fasta' STORED AS FASTA")?;-- let my_statement = my_parser.parse_statement()?;-- match my_statement {- MyStatement::DFStatement(s) => println!("df: {s}"),- MyStatement::MyCopyTo(s) => println!("my_copy: {s}"),- }-- Ok(())-}--/// Here we define a Parser for our new SQL dialect that wraps the existing `DFParser`-struct MyParser<'a> {- df_parser: DFParser<'a>,-}--impl<'a> MyParser<'a> {- fn new(sql: &'a str) -> Result<Self> {- let df_parser = DFParserBuilder::new(sql).build()?;- Ok(Self { df_parser })- }-- /// Returns true if the next token is `COPY` keyword, false otherwise- fn is_copy(&self) -> bool {- matches!(- self.df_parser.parser.peek_token().token,- Token::Word(w) if w.keyword == Keyword::COPY- )- }-- /// This is the entry point to our parser -- it handles `COPY` statements specially- /// but otherwise delegates to the existing DataFusion parser.- pub fn parse_statement(&mut self) -> Result<MyStatement, DataFusionError> {- if self.is_copy() {- self.df_parser.parser.next_token(); // COPY- let df_statement = self.df_parser.parse_copy()?;-- if let Statement::CopyTo(s) = df_statement {- Ok(MyStatement::from(s))- } else {- Ok(MyStatement::DFStatement(Box::from(df_statement)))- }- } else {- let df_statement = self.df_parser.parse_statement()?;- Ok(MyStatement::from(df_statement))- }- }-}--enum MyStatement {- DFStatement(Box<Statement>),- MyCopyTo(MyCopyToStatement),-}--impl Display for MyStatement {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- match self {- MyStatement::DFStatement(s) => write!(f, "{s}"),- MyStatement::MyCopyTo(s) => write!(f, "{s}"),- }- }-}--impl From<Statement> for MyStatement {- fn from(s: Statement) -> Self {- Self::DFStatement(Box::from(s))- }-}--impl From<CopyToStatement> for MyStatement {- fn from(s: CopyToStatement) -> Self {- if s.stored_as == Some("FASTA".to_string()) {- Self::MyCopyTo(MyCopyToStatement::from(s))- } else {- Self::DFStatement(Box::from(Statement::CopyTo(s)))- }- }-}--struct MyCopyToStatement {- pub source: CopyToSource,- pub target: String,-}--impl From<CopyToStatement> for MyCopyToStatement {- fn from(s: CopyToStatement) -> Self {- Self {- source: s.source,- target: s.target,- }- }-}--impl Display for MyCopyToStatement {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(- f,- "COPY {} TO '{}' STORED AS FASTA",- self.source, self.target- )- }-}diff --git a/datafusion-examples/examples/sql_ops/main.rs b/datafusion-examples/examples/sql_ops/main.rsindex 63074e0360..8c3ac05669 100644--- a/datafusion-examples/examples/sql_ops/main.rs+++ b/datafusion-examples/examples/sql_ops/main.rs@@ -21,18 +21,18 @@//!//! ## Usage//! ```bash-//! cargo run --example sql_ops -- [all|analysis|dialect|frontend|query]+//! cargo run --example sql_ops -- [all|analysis|custom_sql_parser|frontend|query]//! ```//!//! Each subcommand runs a corresponding example://! - `all` — run all examples included in this module//! - `analysis` — analyse SQL queries with DataFusion structures-//! - `dialect` — implementing a custom SQL dialect on top of DFParser+//! - `custom_sql_parser` — implementing a custom SQL parser to extend DataFusion//! - `frontend` — create LogicalPlans (only) from sql strings//! - `query` — query data using SQL (in memory RecordBatches, local Parquet files)mod analysis;-mod dialect;+mod custom_sql_parser;mod frontend;mod query;@@ -45,7 +45,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};enum ExampleKind {All,Analysis,- Dialect,+ CustomSqlParser,Frontend,Query,}@@ -66,7 +66,9 @@ impl ExampleKind {}}ExampleKind::Analysis => analysis::analysis().await?,- ExampleKind::Dialect => dialect::dialect().await?,+ ExampleKind::CustomSqlParser => {+ custom_sql_parser::custom_sql_parser().await?+ }ExampleKind::Frontend => frontend::frontend()?,ExampleKind::Query => query::query().await?,}