Skip to content

Apache DataFusion SQL Query Engine

feat: Add builder API for CreateExternalTable to reduce verbosity (#19066)

Aryan Bagade2b05b09b66e6
diff --git a/datafusion/catalog/src/listing_schema.rs b/datafusion/catalog/src/listing_schema.rs
index af96cfc15f..e30366bb3f 100644
--- a/datafusion/catalog/src/listing_schema.rs
+++ b/datafusion/catalog/src/listing_schema.rs
@@ -127,22 +127,13 @@ impl ListingSchemaProvider {
.factory
.create(
state,
- &CreateExternalTable {
- schema: Arc::new(DFSchema::empty()),
+ &CreateExternalTable::builder(
name,
- location: table_url,
- file_type: self.format.clone(),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options: Default::default(),
- constraints: Default::default(),
- column_defaults: Default::default(),
- },
+ table_url,
+ self.format.clone(),
+ Arc::new(DFSchema::empty()),
+ )
+ .build(),
)
.await?;
let _ =
diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs
index 4d35787950..9c5e41ace9 100644
--- a/datafusion/core/src/datasource/listing_table_factory.rs
+++ b/datafusion/core/src/datasource/listing_table_factory.rs
@@ -231,7 +231,7 @@ mod tests {
use std::path::PathBuf;
use datafusion_common::parsers::CompressionTypeVariant;
- use datafusion_common::{Constraints, DFSchema, TableReference};
+ use datafusion_common::{DFSchema, TableReference};
#[tokio::test]
async fn test_create_using_non_std_file_ext() {
@@ -245,22 +245,14 @@ mod tests {
let context = SessionContext::new();
let state = context.state();
let name = TableReference::bare("foo");
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: csv_file.path().to_str().unwrap().to_string(),
- file_type: "csv".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options: HashMap::from([("format.has_header".into(), "true".into())]),
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ csv_file.path().to_str().unwrap().to_string(),
+ "csv",
+ Arc::new(DFSchema::empty()),
+ )
+ .with_options(HashMap::from([("format.has_header".into(), "true".into())]))
+ .build();
let table_provider = factory.create(&state, &cmd).await.unwrap();
let listing_table = table_provider
.as_any()
@@ -286,22 +278,14 @@ mod tests {
let mut options = HashMap::new();
options.insert("format.schema_infer_max_rec".to_owned(), "1000".to_owned());
options.insert("format.has_header".into(), "true".into());
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: csv_file.path().to_str().unwrap().to_string(),
- file_type: "csv".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options,
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ csv_file.path().to_str().unwrap().to_string(),
+ "csv",
+ Arc::new(DFSchema::empty()),
+ )
+ .with_options(options)
+ .build();
let table_provider = factory.create(&state, &cmd).await.unwrap();
let listing_table = table_provider
.as_any()
@@ -331,22 +315,14 @@ mod tests {
options.insert("format.schema_infer_max_rec".to_owned(), "1000".to_owned());
options.insert("format.has_header".into(), "true".into());
options.insert("format.compression".into(), "gzip".into());
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: dir.path().to_str().unwrap().to_string(),
- file_type: "csv".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options,
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ dir.path().to_str().unwrap().to_string(),
+ "csv",
+ Arc::new(DFSchema::empty()),
+ )
+ .with_options(options)
+ .build();
let table_provider = factory.create(&state, &cmd).await.unwrap();
let listing_table = table_provider
.as_any()
@@ -383,22 +359,14 @@ mod tests {
let mut options = HashMap::new();
options.insert("format.schema_infer_max_rec".to_owned(), "1000".to_owned());
options.insert("format.has_header".into(), "true".into());
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: dir.path().to_str().unwrap().to_string(),
- file_type: "csv".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options,
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ dir.path().to_str().unwrap().to_string(),
+ "csv",
+ Arc::new(DFSchema::empty()),
+ )
+ .with_options(options)
+ .build();
let table_provider = factory.create(&state, &cmd).await.unwrap();
let listing_table = table_provider
.as_any()
@@ -427,22 +395,13 @@ mod tests {
let state = context.state();
let name = TableReference::bare("foo");
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: String::from(path.to_str().unwrap()),
- file_type: "parquet".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options: HashMap::new(),
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ String::from(path.to_str().unwrap()),
+ "parquet",
+ Arc::new(DFSchema::empty()),
+ )
+ .build();
let table_provider = factory.create(&state, &cmd).await.unwrap();
let listing_table = table_provider
.as_any()
@@ -467,22 +426,13 @@ mod tests {
let state = context.state();
let name = TableReference::bare("foo");
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: dir.path().to_str().unwrap().to_string(),
- file_type: "parquet".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options: HashMap::new(),
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ dir.path().to_str().unwrap(),
+ "parquet",
+ Arc::new(DFSchema::empty()),
+ )
+ .build();
let table_provider = factory.create(&state, &cmd).await.unwrap();
let listing_table = table_provider
.as_any()
@@ -508,22 +458,13 @@ mod tests {
let state = context.state();
let name = TableReference::bare("foo");
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: dir.path().to_str().unwrap().to_string(),
- file_type: "parquet".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options: HashMap::new(),
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ dir.path().to_str().unwrap().to_string(),
+ "parquet",
+ Arc::new(DFSchema::empty()),
+ )
+ .build();
let table_provider = factory.create(&state, &cmd).await.unwrap();
let listing_table = table_provider
.as_any()
@@ -558,22 +499,13 @@ mod tests {
let state = context.state();
let name = TableReference::bare("test");
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
- location: location.clone(),
- file_type: "parquet".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options: HashMap::new(),
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ location.clone(),
+ "parquet",
+ Arc::new(DFSchema::empty()),
+ )
+ .build();
let _table_provider = factory.create(&state, &cmd).await.unwrap();
@@ -597,22 +529,13 @@ mod tests {
let state = context.state();
let name = TableReference::bare("test");
- let cmd = CreateExternalTable {
+ let cmd = CreateExternalTable::builder(
name,
location,
- file_type: "parquet".to_string(),
- schema: Arc::new(DFSchema::empty()),
- table_partition_cols: vec![],
- if_not_exists: false,
- or_replace: false,
- temporary: false,
- definition: None,
- order_exprs: vec![],
- unbounded: false,
- options: HashMap::new(),
- constraints: Constraints::default(),
- column_defaults: HashMap::new(),
- };
+ "parquet",
+ Arc::new(DFSchema::empty()),
+ )
+ .build();
let _table_provider = factory.create(&state, &cmd).await.unwrap();
diff --git a/datafusion/expr/src/logical_plan/ddl.rs b/datafusion/expr/src/logical_plan/ddl.rs
index ac8ebea17f..8a46e842a8 100644
--- a/datafusion/expr/src/logical_plan/ddl.rs
+++ b/datafusion/expr/src/logical_plan/ddl.rs
@@ -237,6 +237,158 @@ pub struct CreateExternalTable {
pub column_defaults: HashMap<String, Expr>,
}
+impl CreateExternalTable {
+ /// Creates a builder for [`CreateExternalTable`] with required fields.
+ ///
+ /// # Arguments
+ /// * `name` - The table name
+ /// * `location` - The physical location of the table files
+ /// * `file_type` - The file type (e.g., "parquet", "csv", "json")
+ /// * `schema` - The table schema
+ ///
+ /// # Example
+ /// ```
+ /// # use datafusion_expr::CreateExternalTable;
+ /// # use datafusion_common::{DFSchema, TableReference};
+ /// # use std::sync::Arc;
+ /// let table = CreateExternalTable::builder(
+ /// TableReference::bare("my_table"),
+ /// "/path/to/data",
+ /// "parquet",
+ /// Arc::new(DFSchema::empty())
+ /// ).build();
+ /// ```
+ pub fn builder(
+ name: impl Into<TableReference>,
+ location: impl Into<String>,
+ file_type: impl Into<String>,
+ schema: DFSchemaRef,
+ ) -> CreateExternalTableBuilder {
+ CreateExternalTableBuilder {
+ name: name.into(),
+ location: location.into(),
+ file_type: file_type.into(),
+ schema,
+ table_partition_cols: vec![],
+ if_not_exists: false,
+ or_replace: false,
+ temporary: false,
+ definition: None,
+ order_exprs: vec![],
+ unbounded: false,
+ options: HashMap::new(),
+ constraints: Default::default(),
+ column_defaults: HashMap::new(),
+ }
+ }
+}
+
+/// Builder for [`CreateExternalTable`] that provides a fluent API for construction.
+///
+/// Created via [`CreateExternalTable::builder`].
+#[derive(Debug, Clone)]
+pub struct CreateExternalTableBuilder {
+ name: TableReference,
+ location: String,
+ file_type: String,
+ schema: DFSchemaRef,
+ table_partition_cols: Vec<String>,
+ if_not_exists: bool,
+ or_replace: bool,
+ temporary: bool,
+ definition: Option<String>,
+ order_exprs: Vec<Vec<Sort>>,
+ unbounded: bool,
+ options: HashMap<String, String>,
+ constraints: Constraints,
+ column_defaults: HashMap<String, Expr>,
+}
+
+impl CreateExternalTableBuilder {
+ /// Set the partition columns
+ pub fn with_partition_cols(mut self, cols: Vec<String>) -> Self {
+ self.table_partition_cols = cols;
+ self
+ }
+
+ /// Set the if_not_exists flag
+ pub fn with_if_not_exists(mut self, if_not_exists: bool) -> Self {
+ self.if_not_exists = if_not_exists;
+ self
+ }
+
+ /// Set the or_replace flag
+ pub fn with_or_replace(mut self, or_replace: bool) -> Self {
+ self.or_replace = or_replace;
+ self
+ }
+
+ /// Set the temporary flag
+ pub fn with_temporary(mut self, temporary: bool) -> Self {
+ self.temporary = temporary;
+ self
+ }
+
+ /// Set the SQL definition
+ pub fn with_definition(mut self, definition: Option<String>) -> Self {
+ self.definition = definition;
+ self
+ }
+
+ /// Set the order expressions
+ pub fn with_order_exprs(mut self, order_exprs: Vec<Vec<Sort>>) -> Self {
+ self.order_exprs = order_exprs;
+ self
+ }
+
+ /// Set the unbounded flag
+ pub fn with_unbounded(mut self, unbounded: bool) -> Self {
+ self.unbounded = unbounded;
+ self
+ }
+
+ /// Set the table options
+ pub fn with_options(mut self, options: HashMap<String, String>) -> Self {
+ self.options = options;
+ self
+ }
+
+ /// Set the table constraints
+ pub fn with_constraints(mut self, constraints: Constraints) -> Self {
+ self.constraints = constraints;
+ self
+ }
+
+ /// Set the column defaults
+ pub fn with_column_defaults(
+ mut self,
+ column_defaults: HashMap<String, Expr>,
+ ) -> Self {
+ self.column_defaults = column_defaults;
+ self
+ }
+
+ /// Build the [`CreateExternalTable`]
+ pub fn build(self) -> CreateExternalTable {
+ CreateExternalTable {
+ schema: self.schema,
+ name: self.name,
+ location: self.location,
+ file_type: self.file_type,
+ table_partition_cols: self.table_partition_cols,
+ if_not_exists: self.if_not_exists,
+ or_replace: self.or_replace,
+ temporary: self.temporary,
+ definition: self.definition,
+ order_exprs: self.order_exprs,
+ unbounded: self.unbounded,
+ options: self.options,
+ constraints: self.constraints,
+ column_defaults: self.column_defaults,
+ }
+ }
+}
+
// Hashing refers to a subset of fields considered in PartialEq.
impl Hash for CreateExternalTable {
fn hash<H: Hasher>(&self, state: &mut H) {
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index 7a8cbafc22..c42f4d15f2 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -606,27 +606,26 @@ impl AsLogicalPlan for LogicalPlanNode {
}
Ok(LogicalPlan::Ddl(DdlStatement::CreateExternalTable(
- CreateExternalTable {
- schema: pb_schema.try_into()?,
- name: from_table_reference(
+ CreateExternalTable::builder(
+ from_table_reference(
create_extern_table.name.as_ref(),
"CreateExternalTable",
)?,
- location: create_extern_table.location.clone(),
- file_type: create_extern_table.file_type.clone(),
- table_partition_cols: create_extern_table
- .table_partition_cols
- .clone(),
- order_exprs,
- if_not_exists: create_extern_table.if_not_exists,
- or_replace: create_extern_table.or_replace,
- temporary: create_extern_table.temporary,
- definition,
- unbounded: create_extern_table.unbounded,
- options: create_extern_table.options.clone(),
- constraints: constraints.into(),
- column_defaults,
- },
+ create_extern_table.location.clone(),
+ create_extern_table.file_type.clone(),
+ pb_schema.try_into()?,
+ )
+ .with_partition_cols(create_extern_table.table_partition_cols.clone())
+ .with_order_exprs(order_exprs)
+ .with_if_not_exists(create_extern_table.if_not_exists)
+ .with_or_replace(create_extern_table.or_replace)
+ .with_temporary(create_extern_table.temporary)
+ .with_definition(definition)
+ .with_unbounded(create_extern_table.unbounded)
+ .with_options(create_extern_table.options.clone())
+ .with_constraints(constraints.into())
+ .with_column_defaults(column_defaults)
+ .build(),
)))
}
LogicalPlanType::CreateView(create_view) => {
diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index a30c6c31fa..da1e7819f7 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -1643,22 +1643,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
let constraints =
self.new_constraint_from_table_constraints(&all_constraints, &df_schema)?;
Ok(LogicalPlan::Ddl(DdlStatement::CreateExternalTable(
- PlanCreateExternalTable {
- schema: df_schema,
- name,
- location,
- file_type,
- table_partition_cols,
- if_not_exists,
- or_replace,
- temporary,
- definition,
- order_exprs: ordered_exprs,
- unbounded,
- options: options_map,
- constraints,
- column_defaults,
- },
+ PlanCreateExternalTable::builder(name, location, file_type, df_schema)
+ .with_partition_cols(table_partition_cols)
+ .with_if_not_exists(if_not_exists)
+ .with_or_replace(or_replace)
+ .with_temporary(temporary)
+ .with_definition(definition)
+ .with_order_exprs(ordered_exprs)
+ .with_unbounded(unbounded)
+ .with_options(options_map)
+ .with_constraints(constraints)
+ .with_column_defaults(column_defaults)
+ .build(),
)))
}