Skip to content

Apache DataFusion SQL Query Engine

examples: replace sql_dialect with custom_sql_parser example (#19383)

Geoffrey Claudeef2c1a30eb76
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md
index 1469aad541..8f38b38990 100644
--- a/datafusion-examples/README.md
+++ b/datafusion-examples/README.md
@@ -191,12 +191,12 @@ cargo run --example dataframe -- dataframe
#### Category: Single Process
-| Subcommand | File Path | Description |
-| ---------- | ----------------------------------------------------- | ------------------------------ |
-| analysis | [`sql_ops/analysis.rs`](examples/sql_ops/analysis.rs) | Analyze SQL queries |
-| dialect | [`sql_ops/dialect.rs`](examples/sql_ops/dialect.rs) | Implement a custom SQL dialect |
-| frontend | [`sql_ops/frontend.rs`](examples/sql_ops/frontend.rs) | Build LogicalPlans from SQL |
-| query | [`sql_ops/query.rs`](examples/sql_ops/query.rs) | Query data using SQL |
+| Subcommand | File Path | Description |
+| ----------------- | ----------------------------------------------------------------------- | -------------------------------------------------- |
+| analysis | [`sql_ops/analysis.rs`](examples/sql_ops/analysis.rs) | Analyze SQL queries |
+| custom_sql_parser | [`sql_ops/custom_sql_parser.rs`](examples/sql_ops/custom_sql_parser.rs) | Implement a custom SQL parser to extend DataFusion |
+| frontend | [`sql_ops/frontend.rs`](examples/sql_ops/frontend.rs) | Build LogicalPlans from SQL |
+| query | [`sql_ops/query.rs`](examples/sql_ops/query.rs) | Query data using SQL |
## UDF Examples
diff --git a/datafusion-examples/examples/sql_ops/custom_sql_parser.rs b/datafusion-examples/examples/sql_ops/custom_sql_parser.rs
new file mode 100644
index 0000000000..308a0de62a
--- /dev/null
+++ b/datafusion-examples/examples/sql_ops/custom_sql_parser.rs
@@ -0,0 +1,420 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! This example demonstrates extending the DataFusion SQL parser to support
+//! custom DDL statements, specifically `CREATE EXTERNAL CATALOG`.
+//!
+//! ### Custom Syntax
+//! ```sql
+//! CREATE EXTERNAL CATALOG my_catalog
+//! STORED AS ICEBERG
+//! LOCATION 's3://my-bucket/warehouse/'
+//! OPTIONS (
+//! 'region' = 'us-west-2'
+//! );
+//! ```
+//!
+//! Note: For the purpose of this example, we use `local://workspace/` to
+//! automatically discover and register files from the project's test data.
+
+use std::collections::HashMap;
+use std::fmt::Display;
+use std::sync::Arc;
+
+use datafusion::catalog::{
+ CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider,
+ TableProviderFactory,
+};
+use datafusion::datasource::listing_table_factory::ListingTableFactory;
+use datafusion::error::{DataFusionError, Result};
+use datafusion::prelude::SessionContext;
+use datafusion::sql::{
+ parser::{DFParser, DFParserBuilder, Statement},
+ sqlparser::{
+ ast::{ObjectName, Value},
+ keywords::Keyword,
+ tokenizer::Token,
+ },
+};
+use datafusion_common::{DFSchema, TableReference, plan_datafusion_err, plan_err};
+use datafusion_expr::CreateExternalTable;
+use futures::StreamExt;
+use insta::assert_snapshot;
+use object_store::ObjectStore;
+use object_store::local::LocalFileSystem;
+
+/// Entry point for the example.
+pub async fn custom_sql_parser() -> Result<()> {
+ // Use standard Parquet testing data as our "external" source.
+ let base_path = datafusion::common::test_util::parquet_test_data();
+ let base_path = std::path::Path::new(&base_path).canonicalize()?;
+
+ // Make the path relative to the workspace root
+ let workspace_root = workspace_root();
+ let location = base_path
+ .strip_prefix(&workspace_root)
+ .map(|p| p.to_string_lossy().to_string())
+ .unwrap_or_else(|_| base_path.to_string_lossy().to_string());
+
+ let create_catalog_sql = format!(
+ "CREATE EXTERNAL CATALOG parquet_testing
+ STORED AS parquet
+ LOCATION 'local://workspace/{location}'
+ OPTIONS (
+ 'schema_name' = 'staged_data',
+ 'format.pruning' = 'true'
+ )"
+ );
+
+ // =========================================================================
+ // Part 1: Standard DataFusion parser rejects the custom DDL
+ // =========================================================================
+ println!("=== Part 1: Standard DataFusion Parser ===\n");
+ println!("Parsing: {}\n", create_catalog_sql.trim());
+
+ let ctx_standard = SessionContext::new();
+ let err = ctx_standard
+ .sql(&create_catalog_sql)
+ .await
+ .expect_err("Expected the standard parser to reject CREATE EXTERNAL CATALOG (custom DDL syntax)");
+
+ println!("Error: {err}\n");
+ assert_snapshot!(err.to_string(), @r#"SQL error: ParserError("Expected: TABLE, found: CATALOG at Line: 1, Column: 17")"#);
+
+ // =========================================================================
+ // Part 2: Custom parser handles the statement
+ // =========================================================================
+ println!("=== Part 2: Custom Parser ===\n");
+ println!("Parsing: {}\n", create_catalog_sql.trim());
+
+ let ctx = SessionContext::new();
+
+ let mut parser = CustomParser::new(&create_catalog_sql)?;
+ let statement = parser.parse_statement()?;
+ match statement {
+ CustomStatement::CreateExternalCatalog(stmt) => {
+ handle_create_external_catalog(&ctx, stmt).await?;
+ }
+ CustomStatement::DFStatement(_) => {
+ panic!("Expected CreateExternalCatalog statement");
+ }
+ }
+
+ // Query a table from the registered catalog
+ let query_sql = "SELECT id, bool_col, tinyint_col FROM parquet_testing.staged_data.alltypes_plain LIMIT 5";
+ println!("Executing: {query_sql}\n");
+
+ let results = execute_sql(&ctx, query_sql).await?;
+ println!("{results}");
+ assert_snapshot!(results, @r"
+ +----+----------+-------------+
+ | id | bool_col | tinyint_col |
+ +----+----------+-------------+
+ | 4 | true | 0 |
+ | 5 | false | 1 |
+ | 6 | true | 0 |
+ | 7 | false | 1 |
+ | 2 | true | 0 |
+ +----+----------+-------------+
+ ");
+
+ Ok(())
+}
+
+/// Execute SQL and return formatted results.
+async fn execute_sql(ctx: &SessionContext, sql: &str) -> Result<String> {
+ let batches = ctx.sql(sql).await?.collect().await?;
+ Ok(arrow::util::pretty::pretty_format_batches(&batches)?.to_string())
+}
+
+/// Custom handler for the `CREATE EXTERNAL CATALOG` statement.
+async fn handle_create_external_catalog(
+ ctx: &SessionContext,
+ stmt: CreateExternalCatalog,
+) -> Result<()> {
+ let factory = ListingTableFactory::new();
+ let catalog = Arc::new(MemoryCatalogProvider::new());
+ let schema = Arc::new(MemorySchemaProvider::new());
+
+ // Extract options
+ let mut schema_name = "public".to_string();
+ let mut table_options = HashMap::new();
+
+ for (k, v) in stmt.options {
+ let val_str = match v {
+ Value::SingleQuotedString(ref s) | Value::DoubleQuotedString(ref s) => {
+ s.to_string()
+ }
+ Value::Number(ref n, _) => n.to_string(),
+ Value::Boolean(b) => b.to_string(),
+ _ => v.to_string(),
+ };
+
+ if k == "schema_name" {
+ schema_name = val_str;
+ } else {
+ table_options.insert(k, val_str);
+ }
+ }
+
+ println!(" Target Catalog: {}", stmt.name);
+ println!(" Data Location: {}", stmt.location);
+ println!(" Resolved Schema: {schema_name}");
+
+ // Register a local object store rooted at the workspace root.
+ // We use a specific authority 'workspace' to ensure consistent resolution.
+ let store = Arc::new(LocalFileSystem::new_with_prefix(workspace_root())?);
+ let store_url = url::Url::parse("local://workspace").unwrap();
+ ctx.register_object_store(&store_url, Arc::clone(&store) as _);
+
+ let target_ext = format!(".{}", stmt.catalog_type.to_lowercase());
+
+ // For 'local://workspace/parquet-testing/data', the path is 'parquet-testing/data'.
+ let path_str = stmt
+ .location
+ .strip_prefix("local://workspace/")
+ .unwrap_or(&stmt.location);
+ let prefix = object_store::path::Path::from(path_str);
+
+ // Discover data files using the ObjectStore API
+ let mut table_count = 0;
+ let mut list_stream = store.list(Some(&prefix));
+
+ while let Some(meta) = list_stream.next().await {
+ let meta = meta?;
+ let path = &meta.location;
+
+ if path.as_ref().ends_with(&target_ext) {
+ let name = std::path::Path::new(path.as_ref())
+ .file_stem()
+ .unwrap()
+ .to_string_lossy()
+ .to_string();
+
+ let table_url = format!("local://workspace/{path}");
+
+ let cmd = CreateExternalTable::builder(
+ TableReference::bare(name.clone()),
+ table_url,
+ stmt.catalog_type.clone(),
+ Arc::new(DFSchema::empty()),
+ )
+ .with_options(table_options.clone())
+ .build();
+
+ match factory.create(&ctx.state(), &cmd).await {
+ Ok(table) => {
+ schema.register_table(name, table)?;
+ table_count += 1;
+ }
+ Err(e) => {
+ eprintln!("Failed to create table {name}: {e}");
+ }
+ }
+ }
+ }
+ println!(" Registered {table_count} tables into schema: {schema_name}");
+
+ catalog.register_schema(&schema_name, schema)?;
+ ctx.register_catalog(stmt.name.to_string(), catalog);
+
+ Ok(())
+}
+
+/// Possible statements returned by our custom parser.
+#[derive(Debug, Clone)]
+pub enum CustomStatement {
+ /// Standard DataFusion statement
+ DFStatement(Box<Statement>),
+ /// Custom `CREATE EXTERNAL CATALOG` statement
+ CreateExternalCatalog(CreateExternalCatalog),
+}
+
+/// Data structure for `CREATE EXTERNAL CATALOG`.
+#[derive(Debug, Clone)]
+pub struct CreateExternalCatalog {
+ pub name: ObjectName,
+ pub catalog_type: String,
+ pub location: String,
+ pub options: Vec<(String, Value)>,
+}
+
+impl Display for CustomStatement {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::DFStatement(s) => write!(f, "{s}"),
+ Self::CreateExternalCatalog(s) => write!(f, "{s}"),
+ }
+ }
+}
+
+impl Display for CreateExternalCatalog {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "CREATE EXTERNAL CATALOG {} STORED AS {} LOCATION '{}'",
+ self.name, self.catalog_type, self.location
+ )?;
+ if !self.options.is_empty() {
+ write!(f, " OPTIONS (")?;
+ for (i, (k, v)) in self.options.iter().enumerate() {
+ if i > 0 {
+ write!(f, ", ")?;
+ }
+ write!(f, "'{k}' = '{v}'")?;
+ }
+ write!(f, ")")?;
+ }
+ Ok(())
+ }
+}
+
+/// A parser that extends `DFParser` with custom syntax.
+struct CustomParser<'a> {
+ df_parser: DFParser<'a>,
+}
+
+impl<'a> CustomParser<'a> {
+ fn new(sql: &'a str) -> Result<Self> {
+ Ok(Self {
+ df_parser: DFParserBuilder::new(sql).build()?,
+ })
+ }
+
+ pub fn parse_statement(&mut self) -> Result<CustomStatement> {
+ if self.is_create_external_catalog() {
+ return self.parse_create_external_catalog();
+ }
+ Ok(CustomStatement::DFStatement(Box::new(
+ self.df_parser.parse_statement()?,
+ )))
+ }
+
+ fn is_create_external_catalog(&self) -> bool {
+ let t1 = &self.df_parser.parser.peek_nth_token(0).token;
+ let t2 = &self.df_parser.parser.peek_nth_token(1).token;
+ let t3 = &self.df_parser.parser.peek_nth_token(2).token;
+
+ matches!(t1, Token::Word(w) if w.keyword == Keyword::CREATE)
+ && matches!(t2, Token::Word(w) if w.keyword == Keyword::EXTERNAL)
+ && matches!(t3, Token::Word(w) if w.value.to_uppercase() == "CATALOG")
+ }
+
+ fn parse_create_external_catalog(&mut self) -> Result<CustomStatement> {
+ // Consume prefix tokens: CREATE EXTERNAL CATALOG
+ for _ in 0..3 {
+ self.df_parser.parser.next_token();
+ }
+
+ let name = self
+ .df_parser
+ .parser
+ .parse_object_name(false)
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+
+ let mut catalog_type = None;
+ let mut location = None;
+ let mut options = vec![];
+
+ while let Some(keyword) = self.df_parser.parser.parse_one_of_keywords(&[
+ Keyword::STORED,
+ Keyword::LOCATION,
+ Keyword::OPTIONS,
+ ]) {
+ match keyword {
+ Keyword::STORED => {
+ if catalog_type.is_some() {
+ return plan_err!("Duplicate STORED AS");
+ }
+ self.df_parser
+ .parser
+ .expect_keyword(Keyword::AS)
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+ catalog_type = Some(
+ self.df_parser
+ .parser
+ .parse_identifier()
+ .map_err(|e| DataFusionError::External(Box::new(e)))?
+ .value,
+ );
+ }
+ Keyword::LOCATION => {
+ if location.is_some() {
+ return plan_err!("Duplicate LOCATION");
+ }
+ location = Some(
+ self.df_parser
+ .parser
+ .parse_literal_string()
+ .map_err(|e| DataFusionError::External(Box::new(e)))?,
+ );
+ }
+ Keyword::OPTIONS => {
+ if !options.is_empty() {
+ return plan_err!("Duplicate OPTIONS");
+ }
+ options = self.parse_value_options()?;
+ }
+ _ => unreachable!(),
+ }
+ }
+
+ Ok(CustomStatement::CreateExternalCatalog(
+ CreateExternalCatalog {
+ name,
+ catalog_type: catalog_type
+ .ok_or_else(|| plan_datafusion_err!("Missing STORED AS"))?,
+ location: location
+ .ok_or_else(|| plan_datafusion_err!("Missing LOCATION"))?,
+ options,
+ },
+ ))
+ }
+
+ /// Parse options in the form: (key [=] value, key [=] value, ...)
+ fn parse_value_options(&mut self) -> Result<Vec<(String, Value)>> {
+ let mut options = vec![];
+ self.df_parser
+ .parser
+ .expect_token(&Token::LParen)
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+
+ loop {
+ let key = self.df_parser.parse_option_key()?;
+ // Support optional '=' between key and value
+ let _ = self.df_parser.parser.consume_token(&Token::Eq);
+ let value = self.df_parser.parse_option_value()?;
+ options.push((key, value));
+
+ let comma = self.df_parser.parser.consume_token(&Token::Comma);
+ if self.df_parser.parser.consume_token(&Token::RParen) {
+ break;
+ } else if !comma {
+ return plan_err!("Expected ',' or ')' in OPTIONS");
+ }
+ }
+ Ok(options)
+ }
+}
+
+/// Returns the workspace root directory (parent of datafusion-examples).
+fn workspace_root() -> std::path::PathBuf {
+ std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+ .parent()
+ .expect("CARGO_MANIFEST_DIR should have a parent")
+ .to_path_buf()
+}
diff --git a/datafusion-examples/examples/sql_ops/dialect.rs b/datafusion-examples/examples/sql_ops/dialect.rs
deleted file mode 100644
index 62a9f3fa56..0000000000
--- a/datafusion-examples/examples/sql_ops/dialect.rs
+++ /dev/null
@@ -1,135 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! See `main.rs` for how to run it.
-
-use std::fmt::Display;
-
-use datafusion::error::{DataFusionError, Result};
-use datafusion::sql::{
- parser::{CopyToSource, CopyToStatement, DFParser, DFParserBuilder, Statement},
- sqlparser::{keywords::Keyword, tokenizer::Token},
-};
-
-/// This example demonstrates how to use the DFParser to parse a statement in a custom way
-///
-/// This technique can be used to implement a custom SQL dialect, for example.
-pub async fn dialect() -> Result<()> {
- let mut my_parser =
- MyParser::new("COPY source_table TO 'file.fasta' STORED AS FASTA")?;
-
- let my_statement = my_parser.parse_statement()?;
-
- match my_statement {
- MyStatement::DFStatement(s) => println!("df: {s}"),
- MyStatement::MyCopyTo(s) => println!("my_copy: {s}"),
- }
-
- Ok(())
-}
-
-/// Here we define a Parser for our new SQL dialect that wraps the existing `DFParser`
-struct MyParser<'a> {
- df_parser: DFParser<'a>,
-}
-
-impl<'a> MyParser<'a> {
- fn new(sql: &'a str) -> Result<Self> {
- let df_parser = DFParserBuilder::new(sql).build()?;
- Ok(Self { df_parser })
- }
-
- /// Returns true if the next token is `COPY` keyword, false otherwise
- fn is_copy(&self) -> bool {
- matches!(
- self.df_parser.parser.peek_token().token,
- Token::Word(w) if w.keyword == Keyword::COPY
- )
- }
-
- /// This is the entry point to our parser -- it handles `COPY` statements specially
- /// but otherwise delegates to the existing DataFusion parser.
- pub fn parse_statement(&mut self) -> Result<MyStatement, DataFusionError> {
- if self.is_copy() {
- self.df_parser.parser.next_token(); // COPY
- let df_statement = self.df_parser.parse_copy()?;
-
- if let Statement::CopyTo(s) = df_statement {
- Ok(MyStatement::from(s))
- } else {
- Ok(MyStatement::DFStatement(Box::from(df_statement)))
- }
- } else {
- let df_statement = self.df_parser.parse_statement()?;
- Ok(MyStatement::from(df_statement))
- }
- }
-}
-
-enum MyStatement {
- DFStatement(Box<Statement>),
- MyCopyTo(MyCopyToStatement),
-}
-
-impl Display for MyStatement {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- MyStatement::DFStatement(s) => write!(f, "{s}"),
- MyStatement::MyCopyTo(s) => write!(f, "{s}"),
- }
- }
-}
-
-impl From<Statement> for MyStatement {
- fn from(s: Statement) -> Self {
- Self::DFStatement(Box::from(s))
- }
-}
-
-impl From<CopyToStatement> for MyStatement {
- fn from(s: CopyToStatement) -> Self {
- if s.stored_as == Some("FASTA".to_string()) {
- Self::MyCopyTo(MyCopyToStatement::from(s))
- } else {
- Self::DFStatement(Box::from(Statement::CopyTo(s)))
- }
- }
-}
-
-struct MyCopyToStatement {
- pub source: CopyToSource,
- pub target: String,
-}
-
-impl From<CopyToStatement> for MyCopyToStatement {
- fn from(s: CopyToStatement) -> Self {
- Self {
- source: s.source,
- target: s.target,
- }
- }
-}
-
-impl Display for MyCopyToStatement {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(
- f,
- "COPY {} TO '{}' STORED AS FASTA",
- self.source, self.target
- )
- }
-}
diff --git a/datafusion-examples/examples/sql_ops/main.rs b/datafusion-examples/examples/sql_ops/main.rs
index 63074e0360..8c3ac05669 100644
--- a/datafusion-examples/examples/sql_ops/main.rs
+++ b/datafusion-examples/examples/sql_ops/main.rs
@@ -21,18 +21,18 @@
//!
//! ## Usage
//! ```bash
-//! cargo run --example sql_ops -- [all|analysis|dialect|frontend|query]
+//! cargo run --example sql_ops -- [all|analysis|custom_sql_parser|frontend|query]
//! ```
//!
//! Each subcommand runs a corresponding example:
//! - `all` — run all examples included in this module
//! - `analysis` — analyse SQL queries with DataFusion structures
-//! - `dialect` — implementing a custom SQL dialect on top of DFParser
+//! - `custom_sql_parser` — implementing a custom SQL parser to extend DataFusion
//! - `frontend` — create LogicalPlans (only) from sql strings
//! - `query` — query data using SQL (in memory RecordBatches, local Parquet files)
mod analysis;
-mod dialect;
+mod custom_sql_parser;
mod frontend;
mod query;
@@ -45,7 +45,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};
enum ExampleKind {
All,
Analysis,
- Dialect,
+ CustomSqlParser,
Frontend,
Query,
}
@@ -66,7 +66,9 @@ impl ExampleKind {
}
}
ExampleKind::Analysis => analysis::analysis().await?,
- ExampleKind::Dialect => dialect::dialect().await?,
+ ExampleKind::CustomSqlParser => {
+ custom_sql_parser::custom_sql_parser().await?
+ }
ExampleKind::Frontend => frontend::frontend()?,
ExampleKind::Query => query::query().await?,
}