Apache DataFusion SQL Query Engine
feat: Add builder API for CreateExternalTable to reduce verbosity (#19066)
Aryan Bagade2b05b09b66e6
diff --git a/datafusion/catalog/src/listing_schema.rs b/datafusion/catalog/src/listing_schema.rsindex af96cfc15f..e30366bb3f 100644--- a/datafusion/catalog/src/listing_schema.rs+++ b/datafusion/catalog/src/listing_schema.rs@@ -127,22 +127,13 @@ impl ListingSchemaProvider {.factory.create(state,- &CreateExternalTable {- schema: Arc::new(DFSchema::empty()),+ &CreateExternalTable::builder(name,- location: table_url,- file_type: self.format.clone(),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options: Default::default(),- constraints: Default::default(),- column_defaults: Default::default(),- },+ table_url,+ self.format.clone(),+ Arc::new(DFSchema::empty()),+ )+ .build(),).await?;let _ =diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rsindex 4d35787950..9c5e41ace9 100644--- a/datafusion/core/src/datasource/listing_table_factory.rs+++ b/datafusion/core/src/datasource/listing_table_factory.rs@@ -231,7 +231,7 @@ mod tests {use std::path::PathBuf;use datafusion_common::parsers::CompressionTypeVariant;- use datafusion_common::{Constraints, DFSchema, TableReference};+ use datafusion_common::{DFSchema, TableReference};#[tokio::test]async fn test_create_using_non_std_file_ext() {@@ -245,22 +245,14 @@ mod tests {let context = SessionContext::new();let state = context.state();let name = TableReference::bare("foo");- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: csv_file.path().to_str().unwrap().to_string(),- file_type: "csv".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options: HashMap::from([("format.has_header".into(), "true".into())]),- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ csv_file.path().to_str().unwrap().to_string(),+ "csv",+ Arc::new(DFSchema::empty()),+ )+ .with_options(HashMap::from([("format.has_header".into(), "true".into())]))+ .build();let table_provider = factory.create(&state, &cmd).await.unwrap();let listing_table = table_provider.as_any()@@ -286,22 +278,14 @@ mod tests {let mut options = HashMap::new();options.insert("format.schema_infer_max_rec".to_owned(), "1000".to_owned());options.insert("format.has_header".into(), "true".into());- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: csv_file.path().to_str().unwrap().to_string(),- file_type: "csv".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options,- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ csv_file.path().to_str().unwrap().to_string(),+ "csv",+ Arc::new(DFSchema::empty()),+ )+ .with_options(options)+ .build();let table_provider = factory.create(&state, &cmd).await.unwrap();let listing_table = table_provider.as_any()@@ -331,22 +315,14 @@ mod tests {options.insert("format.schema_infer_max_rec".to_owned(), "1000".to_owned());options.insert("format.has_header".into(), "true".into());options.insert("format.compression".into(), "gzip".into());- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: dir.path().to_str().unwrap().to_string(),- file_type: "csv".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options,- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ dir.path().to_str().unwrap().to_string(),+ "csv",+ Arc::new(DFSchema::empty()),+ )+ .with_options(options)+ .build();let table_provider = factory.create(&state, &cmd).await.unwrap();let listing_table = table_provider.as_any()@@ -383,22 +359,14 @@ mod tests {let mut options = HashMap::new();options.insert("format.schema_infer_max_rec".to_owned(), "1000".to_owned());options.insert("format.has_header".into(), "true".into());- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: dir.path().to_str().unwrap().to_string(),- file_type: "csv".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options,- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ dir.path().to_str().unwrap().to_string(),+ "csv",+ Arc::new(DFSchema::empty()),+ )+ .with_options(options)+ .build();let table_provider = factory.create(&state, &cmd).await.unwrap();let listing_table = table_provider.as_any()@@ -427,22 +395,13 @@ mod tests {let state = context.state();let name = TableReference::bare("foo");- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: String::from(path.to_str().unwrap()),- file_type: "parquet".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options: HashMap::new(),- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ String::from(path.to_str().unwrap()),+ "parquet",+ Arc::new(DFSchema::empty()),+ )+ .build();let table_provider = factory.create(&state, &cmd).await.unwrap();let listing_table = table_provider.as_any()@@ -467,22 +426,13 @@ mod tests {let state = context.state();let name = TableReference::bare("foo");- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: dir.path().to_str().unwrap().to_string(),- file_type: "parquet".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options: HashMap::new(),- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ dir.path().to_str().unwrap(),+ "parquet",+ Arc::new(DFSchema::empty()),+ )+ .build();let table_provider = factory.create(&state, &cmd).await.unwrap();let listing_table = table_provider.as_any()@@ -508,22 +458,13 @@ mod tests {let state = context.state();let name = TableReference::bare("foo");- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: dir.path().to_str().unwrap().to_string(),- file_type: "parquet".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options: HashMap::new(),- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ dir.path().to_str().unwrap().to_string(),+ "parquet",+ Arc::new(DFSchema::empty()),+ )+ .build();let table_provider = factory.create(&state, &cmd).await.unwrap();let listing_table = table_provider.as_any()@@ -558,22 +499,13 @@ mod tests {let state = context.state();let name = TableReference::bare("test");- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,- location: location.clone(),- file_type: "parquet".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options: HashMap::new(),- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ location.clone(),+ "parquet",+ Arc::new(DFSchema::empty()),+ )+ .build();let _table_provider = factory.create(&state, &cmd).await.unwrap();@@ -597,22 +529,13 @@ mod tests {let state = context.state();let name = TableReference::bare("test");- let cmd = CreateExternalTable {+ let cmd = CreateExternalTable::builder(name,location,- file_type: "parquet".to_string(),- schema: Arc::new(DFSchema::empty()),- table_partition_cols: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- definition: None,- order_exprs: vec![],- unbounded: false,- options: HashMap::new(),- constraints: Constraints::default(),- column_defaults: HashMap::new(),- };+ "parquet",+ Arc::new(DFSchema::empty()),+ )+ .build();let _table_provider = factory.create(&state, &cmd).await.unwrap();diff --git a/datafusion/expr/src/logical_plan/ddl.rs b/datafusion/expr/src/logical_plan/ddl.rsindex ac8ebea17f..8a46e842a8 100644--- a/datafusion/expr/src/logical_plan/ddl.rs+++ b/datafusion/expr/src/logical_plan/ddl.rs@@ -237,6 +237,158 @@ pub struct CreateExternalTable {pub column_defaults: HashMap<String, Expr>,}+impl CreateExternalTable {+ /// Creates a builder for [`CreateExternalTable`] with required fields.+ ///+ /// # Arguments+ /// * `name` - The table name+ /// * `location` - The physical location of the table files+ /// * `file_type` - The file type (e.g., "parquet", "csv", "json")+ /// * `schema` - The table schema+ ///+ /// # Example+ /// ```+ /// # use datafusion_expr::CreateExternalTable;+ /// # use datafusion_common::{DFSchema, TableReference};+ /// # use std::sync::Arc;+ /// let table = CreateExternalTable::builder(+ /// TableReference::bare("my_table"),+ /// "/path/to/data",+ /// "parquet",+ /// Arc::new(DFSchema::empty())+ /// ).build();+ /// ```+ pub fn builder(+ name: impl Into<TableReference>,+ location: impl Into<String>,+ file_type: impl Into<String>,+ schema: DFSchemaRef,+ ) -> CreateExternalTableBuilder {+ CreateExternalTableBuilder {+ name: name.into(),+ location: location.into(),+ file_type: file_type.into(),+ schema,+ table_partition_cols: vec![],+ if_not_exists: false,+ or_replace: false,+ temporary: false,+ definition: None,+ order_exprs: vec![],+ unbounded: false,+ options: HashMap::new(),+ constraints: Default::default(),+ column_defaults: HashMap::new(),+ }+ }+}++/// Builder for [`CreateExternalTable`] that provides a fluent API for construction.+///+/// Created via [`CreateExternalTable::builder`].+#[derive(Debug, Clone)]+pub struct CreateExternalTableBuilder {+ name: TableReference,+ location: String,+ file_type: String,+ schema: DFSchemaRef,+ table_partition_cols: Vec<String>,+ if_not_exists: bool,+ or_replace: bool,+ temporary: bool,+ definition: Option<String>,+ order_exprs: Vec<Vec<Sort>>,+ unbounded: bool,+ options: HashMap<String, String>,+ constraints: Constraints,+ column_defaults: HashMap<String, Expr>,+}++impl CreateExternalTableBuilder {+ /// Set the partition columns+ pub fn with_partition_cols(mut self, cols: Vec<String>) -> Self {+ self.table_partition_cols = cols;+ self+ }++ /// Set the if_not_exists flag+ pub fn with_if_not_exists(mut self, if_not_exists: bool) -> Self {+ self.if_not_exists = if_not_exists;+ self+ }++ /// Set the or_replace flag+ pub fn with_or_replace(mut self, or_replace: bool) -> Self {+ self.or_replace = or_replace;+ self+ }++ /// Set the temporary flag+ pub fn with_temporary(mut self, temporary: bool) -> Self {+ self.temporary = temporary;+ self+ }++ /// Set the SQL definition+ pub fn with_definition(mut self, definition: Option<String>) -> Self {+ self.definition = definition;+ self+ }++ /// Set the order expressions+ pub fn with_order_exprs(mut self, order_exprs: Vec<Vec<Sort>>) -> Self {+ self.order_exprs = order_exprs;+ self+ }++ /// Set the unbounded flag+ pub fn with_unbounded(mut self, unbounded: bool) -> Self {+ self.unbounded = unbounded;+ self+ }++ /// Set the table options+ pub fn with_options(mut self, options: HashMap<String, String>) -> Self {+ self.options = options;+ self+ }++ /// Set the table constraints+ pub fn with_constraints(mut self, constraints: Constraints) -> Self {+ self.constraints = constraints;+ self+ }++ /// Set the column defaults+ pub fn with_column_defaults(+ mut self,+ column_defaults: HashMap<String, Expr>,+ ) -> Self {+ self.column_defaults = column_defaults;+ self+ }++ /// Build the [`CreateExternalTable`]+ pub fn build(self) -> CreateExternalTable {+ CreateExternalTable {+ schema: self.schema,+ name: self.name,+ location: self.location,+ file_type: self.file_type,+ table_partition_cols: self.table_partition_cols,+ if_not_exists: self.if_not_exists,+ or_replace: self.or_replace,+ temporary: self.temporary,+ definition: self.definition,+ order_exprs: self.order_exprs,+ unbounded: self.unbounded,+ options: self.options,+ constraints: self.constraints,+ column_defaults: self.column_defaults,+ }+ }+}+// Hashing refers to a subset of fields considered in PartialEq.impl Hash for CreateExternalTable {fn hash<H: Hasher>(&self, state: &mut H) {diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 7a8cbafc22..c42f4d15f2 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -606,27 +606,26 @@ impl AsLogicalPlan for LogicalPlanNode {}Ok(LogicalPlan::Ddl(DdlStatement::CreateExternalTable(- CreateExternalTable {- schema: pb_schema.try_into()?,- name: from_table_reference(+ CreateExternalTable::builder(+ from_table_reference(create_extern_table.name.as_ref(),"CreateExternalTable",)?,- location: create_extern_table.location.clone(),- file_type: create_extern_table.file_type.clone(),- table_partition_cols: create_extern_table- .table_partition_cols- .clone(),- order_exprs,- if_not_exists: create_extern_table.if_not_exists,- or_replace: create_extern_table.or_replace,- temporary: create_extern_table.temporary,- definition,- unbounded: create_extern_table.unbounded,- options: create_extern_table.options.clone(),- constraints: constraints.into(),- column_defaults,- },+ create_extern_table.location.clone(),+ create_extern_table.file_type.clone(),+ pb_schema.try_into()?,+ )+ .with_partition_cols(create_extern_table.table_partition_cols.clone())+ .with_order_exprs(order_exprs)+ .with_if_not_exists(create_extern_table.if_not_exists)+ .with_or_replace(create_extern_table.or_replace)+ .with_temporary(create_extern_table.temporary)+ .with_definition(definition)+ .with_unbounded(create_extern_table.unbounded)+ .with_options(create_extern_table.options.clone())+ .with_constraints(constraints.into())+ .with_column_defaults(column_defaults)+ .build(),)))}LogicalPlanType::CreateView(create_view) => {diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex a30c6c31fa..da1e7819f7 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -1643,22 +1643,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let constraints =self.new_constraint_from_table_constraints(&all_constraints, &df_schema)?;Ok(LogicalPlan::Ddl(DdlStatement::CreateExternalTable(- PlanCreateExternalTable {- schema: df_schema,- name,- location,- file_type,- table_partition_cols,- if_not_exists,- or_replace,- temporary,- definition,- order_exprs: ordered_exprs,- unbounded,- options: options_map,- constraints,- column_defaults,- },+ PlanCreateExternalTable::builder(name, location, file_type, df_schema)+ .with_partition_cols(table_partition_cols)+ .with_if_not_exists(if_not_exists)+ .with_or_replace(or_replace)+ .with_temporary(temporary)+ .with_definition(definition)+ .with_order_exprs(ordered_exprs)+ .with_unbounded(unbounded)+ .with_options(options_map)+ .with_constraints(constraints)+ .with_column_defaults(column_defaults)+ .build(),)))}