Apache DataFusion SQL Query Engine
feat: Support multiple external table locations (#22695)
Kumar Ujjawal1dc031d95e16
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rsindex f43854821b..bc2c15f48d 100644--- a/datafusion-cli/src/exec.rs+++ b/datafusion-cli/src/exec.rs@@ -423,16 +423,17 @@ async fn create_plan(// Expose stdin (e.g. `cat data.csv | datafusion-cli`) as a `stdin://`// object store, registered like any other scheme in `get_object_store`.- cmd.location = StdinUtils::rewrite_location(&cmd.location, format.as_ref());-- register_object_store_and_config_extensions(- ctx,- &cmd.location,- &cmd.options,- format,- resolve_region,- )- .await?;+ for location in &mut cmd.locations {+ *location = StdinUtils::rewrite_location(location, format.as_ref());+ register_object_store_and_config_extensions(+ ctx,+ location,+ &cmd.options,+ format.clone(),+ resolve_region,+ )+ .await?;+ }}if let LogicalPlan::Copy(copy_to) = &mut plan {@@ -535,14 +536,16 @@ mod tests {if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &plan {let format = config_file_type_from_str(&cmd.file_type);- register_object_store_and_config_extensions(- &ctx,- &cmd.location,- &cmd.options,- format,- false,- )- .await?;+ for location in &cmd.locations {+ register_object_store_and_config_extensions(+ &ctx,+ location,+ &cmd.options,+ format.clone(),+ false,+ )+ .await?;+ }} else {return plan_err!("LogicalPlan is not a CreateExternalTable");}diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex ce739e5019..fabc7ca2a0 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -54,7 +54,7 @@ use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::empty::EmptyExec;use futures::{Stream, StreamExt, TryStreamExt, future, stream};use object_store::ObjectStore;-use std::collections::HashMap;+use std::collections::{HashMap, HashSet};use std::sync::Arc;/// Result of a file listing operation from [`ListingTable::list_files_for_scan`].@@ -816,7 +816,15 @@ impl ListingTable {.await?;let meta_fetch_concurrency =ctx.config_options().execution.meta_fetch_concurrency.get();- let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency);+ // Table paths can overlap, for example when one path is a directory and+ // another names a file inside it. A ListingTable uses one object store,+ // so the object path uniquely identifies a file within this scan.+ let mut seen_files = HashSet::new();+ let file_list = stream::iter(file_list)+ .flatten_unordered(meta_fetch_concurrency)+ .try_filter(move |file| {+ future::ready(seen_files.insert(file.object_meta.location.clone()))+ });// collect the statistics and ordering if required by the configlet files = file_list.map(|part_file| async {diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rsindex 8501ea6590..c8060456dd 100644--- a/datafusion/catalog/src/stream.rs+++ b/datafusion/catalog/src/stream.rs@@ -53,7 +53,15 @@ impl TableProviderFactory for StreamTableFactory {cmd: &CreateExternalTable,) -> Result<Arc<dyn TableProvider>> {let schema: SchemaRef = Arc::clone(cmd.schema.inner());- let location = cmd.location.clone();+ let location = match cmd.locations.as_slice() {+ [single] => single.clone(),+ _ => {+ return config_err!(+ "Stream tables support exactly one location; \+ use a listing table to read multiple files"+ );+ }+ };let encoding = cmd.file_type.parse()?;let header = if let Ok(opt) = cmd.optionsdiff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rsindex 3733fb8be6..ba94d23236 100644--- a/datafusion/core/src/datasource/listing_table_factory.rs+++ b/datafusion/core/src/datasource/listing_table_factory.rs@@ -27,9 +27,11 @@ use crate::datasource::listing::{};use crate::execution::context::SessionState;-use arrow::datatypes::DataType;+use arrow::datatypes::{DataType, SchemaRef};use datafusion_common::{Result, config_datafusion_err};-use datafusion_common::{ToDFSchema, arrow_datafusion_err, plan_err};+use datafusion_common::{+ ToDFSchema, arrow_datafusion_err, internal_datafusion_err, plan_err,+};use datafusion_expr::CreateExternalTable;use async_trait::async_trait;@@ -71,19 +73,66 @@ impl TableProviderFactory for ListingTableFactory {))?.create(session_state, &cmd.options)?;- let mut table_path =- ListingTableUrl::parse(&cmd.location)?.with_table_ref(cmd.name.clone());- let file_extension = match table_path.is_collection() {- // Setting the extension to be empty instead of allowing the default extension seems- // odd, but was done to ensure existing behavior isn't modified. It seems like this- // could be refactored to either use the default extension or set the fully expected- // extension when compression is included (e.g. ".csv.gz")- true => "",- false => &get_extension(cmd.location.as_str()),+ let table_paths = cmd+ .locations+ .iter()+ .map(|location| {+ Ok(ListingTableUrl::parse(location)?.with_table_ref(cmd.name.clone()))+ })+ .collect::<Result<Vec<_>>>()?;+ let Some(first_path) = table_paths.first() else {+ return plan_err!("CREATE EXTERNAL TABLE requires at least one location");+ };++ let mut seen_paths = HashSet::with_capacity(table_paths.len());+ if let Some(duplicate) = table_paths.iter().find(|path| !seen_paths.insert(*path))+ {+ return plan_err!(+ "Duplicate location '{}' in CREATE EXTERNAL TABLE",+ duplicate.as_str()+ );+ }++ // `ListingTable` resolves a single object store (from the first location)+ // and scans every location with it, so locations spanning different+ // object stores would silently read the wrong data. Reading across+ // object stores is intentionally not supported (see+ // https://github.com/apache/datafusion/issues/16303); reject it here with+ // a clear error rather than producing incorrect results at scan time.+ let object_store_url = first_path.object_store();+ if let Some(other) = table_paths+ .iter()+ .find(|path| path.object_store() != object_store_url)+ {+ return plan_err!(+ "All locations of a CREATE EXTERNAL TABLE must be on the same \+ object store, but found '{}' and '{}'",+ object_store_url.as_str(),+ other.object_store().as_str()+ );+ }++ // With a single location the historical extension handling is kept. With+ // more than one location the files may have different extensions, so the+ // extension filter is left empty and the explicit paths/globs are used+ // as provided.+ let file_extension = if table_paths.len() == 1 {+ match first_path.is_collection() {+ // Setting the extension to be empty instead of allowing the default extension seems+ // odd, but was done to ensure existing behavior isn't modified. It seems like this+ // could be refactored to either use the default extension or set the fully expected+ // extension when compression is included (e.g. ".csv.gz")+ true => String::new(),+ false => get_extension(&cmd.locations[0]),+ }+ } else {+ String::new()};let mut options =ListingOptions::new(file_format).with_file_extension(file_extension);+ // Partition columns are derived from the first location; all locations+ // are expected to share the same partitioning.let (provided_schema, table_partition_cols) = if cmd.schema.fields().is_empty() {let infer_parts = session_state.config_options()@@ -91,7 +140,7 @@ impl TableProviderFactory for ListingTableFactory {.listing_table_factory_infer_partitions;let part_cols = if cmd.table_partition_cols.is_empty() && infer_parts {options- .infer_partitions(session_state, &table_path)+ .infer_partitions(session_state, first_path).await?.into_iter()} else {@@ -141,36 +190,75 @@ impl TableProviderFactory for ListingTableFactory {options = options.with_table_partition_cols(table_partition_cols);- options- .validate_partitions(session_state, &table_path)- .await?;+ // Validate partitions against every location before any glob rewriting.+ for table_path in &table_paths {+ options+ .validate_partitions(session_state, table_path)+ .await?;+ }- let resolved_schema = match provided_schema {+ let (resolved_table_paths, resolved_schema) = match provided_schema {// We will need to check the table columns against the schema// this is done so that we can do an ORDER BY for external table creation// specifically for parquet file format.// See: https://github.com/apache/datafusion/issues/7317None => {- // if the folder then rewrite a file path as 'path/*.parquet'- // to only read the files the reader can understand- if table_path.is_folder() && table_path.get_glob().is_none() {- // Since there are no files yet to infer an actual extension,- // derive the pattern based on compression type.- // So for gzipped CSV the pattern is `*.csv.gz`- let glob = match options.format.compression_type() {- Some(compression) => {- match options.format.get_ext_with_compression(&compression) {- // Use glob based on `FileFormat` extension- Ok(ext) => format!("*.{ext}"),- // Fallback to `file_type`, if not supported by `FileFormat`- Err(_) => format!("*.{}", cmd.file_type.to_lowercase()),+ let mut resolved_paths = Vec::with_capacity(table_paths.len());+ let mut inferred_schema: Option<(String, SchemaRef)> = None;+ for mut table_path in table_paths {+ // if the folder then rewrite a file path as 'path/*.parquet'+ // to only read the files the reader can understand+ if table_path.is_folder() && table_path.get_glob().is_none() {+ // Since there are no files yet to infer an actual extension,+ // derive the pattern based on compression type.+ // So for gzipped CSV the pattern is `*.csv.gz`+ let glob = match options.format.compression_type() {+ Some(compression) => {+ match options+ .format+ .get_ext_with_compression(&compression)+ {+ // Use glob based on `FileFormat` extension+ Ok(ext) => format!("*.{ext}"),+ // Fallback to `file_type`, if not supported by `FileFormat`+ Err(_) => {+ format!("*.{}", cmd.file_type.to_lowercase())+ }+ }}+ None => format!("*.{}", cmd.file_type.to_lowercase()),+ };+ table_path = table_path.with_glob(glob.as_ref())?;+ }+ let schema = options.infer_schema(session_state, &table_path).await?;+ // All locations must resolve to the same fields. Schema+ // and field metadata may differ between files without+ // changing the fields read by the table.+ let location = table_path.to_string();+ match &inferred_schema {+ None => inferred_schema = Some((location, schema)),+ Some((existing_location, existing))+ if !schemas_have_same_fields(existing, &schema) =>+ {+ return plan_err!(+ "All locations of a CREATE EXTERNAL TABLE must have the \+ same schema, but schema inferred from '{}' differs from \+ schema inferred from '{}'",+ location,+ existing_location+ );}- None => format!("*.{}", cmd.file_type.to_lowercase()),- };- table_path = table_path.with_glob(glob.as_ref())?;+ Some(_) => {}+ }+ resolved_paths.push(table_path);}- let schema = options.infer_schema(session_state, &table_path).await?;+ // `table_paths` was guaranteed non-empty above, so the loop ran+ // at least once and `inferred_schema` is always `Some` here.+ let (_, schema) = inferred_schema.ok_or_else(|| {+ internal_datafusion_err!(+ "no schema could be inferred from the provided locations"+ )+ })?;let df_schema = Arc::clone(&schema).to_dfschema()?;let column_refs: HashSet<_> = cmd.order_exprs@@ -185,11 +273,11 @@ impl TableProviderFactory for ListingTableFactory {}}- schema+ (resolved_paths, schema)}- Some(s) => s,+ Some(s) => (table_paths, s),};- let config = ListingTableConfig::new(table_path)+ let config = ListingTableConfig::new_with_multi_paths(resolved_table_paths).with_listing_options(options.with_file_sort_order(cmd.order_exprs.clone())).with_schema(resolved_schema);let provider = ListingTable::try_new(config)?@@ -221,6 +309,19 @@ fn get_extension(path: &str) -> String {}}+fn schemas_have_same_fields(left: &SchemaRef, right: &SchemaRef) -> bool {+ left.fields().len() == right.fields().len()+ && left+ .fields()+ .iter()+ .zip(right.fields())+ .all(|(left, right)| {+ left.name() == right.name()+ && left.data_type() == right.data_type()+ && left.is_nullable() == right.is_nullable()+ })+}+#[cfg(test)]mod tests {use super::*;@@ -228,6 +329,7 @@ mod tests {datasource::file_format::csv::CsvFormat, execution::context::SessionContext,test_util::parquet_test_data,};+ use arrow::datatypes::{Field, Schema};use datafusion_execution::cache::cache_manager::{CacheManagerConfig, DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,};@@ -237,7 +339,7 @@ mod tests {use std::collections::HashMap;use std::fs;use std::fs::File;- use std::path::PathBuf;+ use std::path::{Path, PathBuf};use datafusion_common::parsers::CompressionTypeVariant;use datafusion_common::{DFSchema, TableReference};@@ -245,6 +347,42 @@ mod tests {use datafusion_execution::cache::default_cache::DefaultCache;use datafusion_expr::registry::ExtensionTypeRegistryRef;+ fn factory_and_state() -> (ListingTableFactory, SessionState) {+ let factory = ListingTableFactory::new();+ let context = SessionContext::new();+ let state = context.state();+ (factory, state)+ }++ fn write_csv(path: &Path, contents: &str) {+ fs::write(path, contents).unwrap();+ }++ fn csv_cmd_with_locations(paths: &[&Path]) -> CreateExternalTable {+ let locations = paths+ .iter()+ .map(|path| path.to_str().unwrap().to_string())+ .collect::<Vec<_>>();++ CreateExternalTable::builder(+ TableReference::bare("foo"),+ locations[0].clone(),+ "csv",+ Arc::new(DFSchema::empty()),+ )+ .with_locations(locations)+ .with_options(HashMap::from([("format.has_header".into(), "true".into())]))+ .build()+ }++ fn assert_error_contains(error: impl std::fmt::Display, expected: &str) {+ let error = error.to_string();+ assert!(+ error.contains(expected),+ "expected error to contain '{expected}', got: {error}"+ );+ }+#[tokio::test]async fn test_create_using_non_std_file_ext() {let csv_file = tempfile::Builder::new()@@ -475,6 +613,151 @@ mod tests {assert!(listing_options.table_partition_cols.is_empty());}+ #[tokio::test]+ async fn test_create_with_multiple_locations() {+ let dir = tempfile::tempdir().unwrap();+ let file_a = dir.path().join("file_a.csv");+ let file_b = dir.path().join("file_b.csv");+ write_csv(&file_a, "c1,c2\n1,a\n2,b\n");+ write_csv(&file_b, "c1,c2\n3,c\n");++ let (factory, state) = factory_and_state();+ let cmd = csv_cmd_with_locations(&[&file_a, &file_b]);++ let table_provider = factory.create(&state, &cmd).await.unwrap();+ let listing_table = table_provider.downcast_ref::<ListingTable>().unwrap();++ // Both locations are registered as table paths+ assert_eq!(2, listing_table.table_paths().len());++ // Schema is inferred from the files and shared across both locations+ let field_names: Vec<_> = listing_table+ .schema()+ .fields()+ .iter()+ .map(|f| f.name().clone())+ .collect();+ assert_eq!(field_names, vec!["c1".to_string(), "c2".to_string()]);+ }++ #[tokio::test]+ async fn test_create_with_duplicate_locations_errors() {+ let dir = tempfile::tempdir().unwrap();+ let file = dir.path().join("file.csv");+ write_csv(&file, "c1,c2\n1,a\n");++ let (factory, state) = factory_and_state();+ let cmd = csv_cmd_with_locations(&[&file, &file]);+ let err = factory.create(&state, &cmd).await.unwrap_err();+ assert_error_contains(err, "Duplicate location");+ }++ #[tokio::test]+ async fn test_create_with_overlapping_locations_reads_each_file_once() {+ let dir = tempfile::tempdir().unwrap();+ let file_a = dir.path().join("file_a.csv");+ let file_b = dir.path().join("file_b.csv");+ write_csv(&file_a, "c1,c2\n1,a\n");+ write_csv(&file_b, "c1,c2\n2,b\n");++ let (factory, state) = factory_and_state();+ let cmd = csv_cmd_with_locations(&[dir.path(), file_a.as_path()]);+ let table_provider = factory.create(&state, &cmd).await.unwrap();+ let listing_table = table_provider.downcast_ref::<ListingTable>().unwrap();++ let listed_files = listing_table+ .list_files_for_scan(&state, &[], None)+ .await+ .unwrap()+ .file_groups+ .iter()+ .map(|group| group.len())+ .sum::<usize>();+ assert_eq!(listed_files, 2);+ }++ #[tokio::test]+ async fn test_create_with_multiple_locations_mismatched_schema_errors() {+ let dir = tempfile::tempdir().unwrap();+ let file_a = dir.path().join("file_a.csv");+ let file_b = dir.path().join("file_b.csv");+ write_csv(&file_a, "c1,c2\n1,a\n");+ // Different column names -> different inferred schema+ write_csv(&file_b, "x1,x2\n1,a\n");++ let (factory, state) = factory_and_state();+ let cmd = csv_cmd_with_locations(&[&file_a, &file_b]);+ let err = factory.create(&state, &cmd).await.unwrap_err();+ assert_error_contains(err, "same schema");+ }++ #[test]+ fn test_schema_comparison_ignores_schema_metadata() {+ let fields =+ vec![+ Field::new("c1", DataType::Int32, true).with_metadata(HashMap::from([(+ "field_source".to_string(),+ "a".to_string(),+ )])),+ ];+ let schema_a = Arc::new(Schema::new_with_metadata(+ fields.clone(),+ HashMap::from([("source".to_string(), "a".to_string())]),+ ));+ let schema_b =+ Arc::new(Schema::new_with_metadata(+ vec![Field::new("c1", DataType::Int32, true).with_metadata(+ HashMap::from([("field_source".to_string(), "b".to_string())]),+ )],+ HashMap::from([("source".to_string(), "b".to_string())]),+ ));+ let schema_c =+ Arc::new(Schema::new(vec![Field::new("c2", DataType::Int32, true)]));++ assert_ne!(schema_a, schema_b);+ assert!(schemas_have_same_fields(&schema_a, &schema_b));+ assert!(!schemas_have_same_fields(&schema_a, &schema_c));+ }++ #[tokio::test]+ async fn test_create_with_no_locations_errors() {+ let (factory, state) = factory_and_state();++ let cmd = CreateExternalTable::builder(+ TableReference::bare("foo"),+ "unused",+ "csv",+ Arc::new(DFSchema::empty()),+ )+ .with_locations(vec![])+ .build();++ let err = factory.create(&state, &cmd).await.unwrap_err();+ assert_error_contains(err, "at least one location");+ }++ #[tokio::test]+ async fn test_create_with_locations_on_different_stores_errors() {+ let (factory, state) = factory_and_state();++ // Two locations on different object stores (different buckets) are not+ // supported: ListingTable would scan both against the first store.+ let cmd = CreateExternalTable::builder(+ TableReference::bare("foo"),+ "s3://bucket_a/file.parquet",+ "parquet",+ Arc::new(DFSchema::empty()),+ )+ .with_locations(vec![+ "s3://bucket_a/file.parquet".to_string(),+ "s3://bucket_b/file.parquet".to_string(),+ ])+ .build();++ let err = factory.create(&state, &cmd).await.unwrap_err();+ assert_error_contains(err, "same object store");+ }+#[tokio::test]async fn test_statistics_cache_prewarming() {let factory = ListingTableFactory::new();diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rsindex aad659eacb..d70c0d186d 100644--- a/datafusion/core/src/test_util/mod.rs+++ b/datafusion/core/src/test_util/mod.rs@@ -45,7 +45,7 @@ use crate::execution::{SendableRecordBatchStream, SessionState, SessionStateBuiluse arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_catalog::Session;-use datafusion_common::{DFSchemaRef, TableReference};+use datafusion_common::{DFSchemaRef, TableReference, plan_err};use datafusion_expr::{CreateExternalTable, Expr, LogicalPlan, SortExpr, TableType,UserDefinedLogicalNodeCore,@@ -187,8 +187,12 @@ impl TableProviderFactory for TestTableFactory {_: &dyn Session,cmd: &CreateExternalTable,) -> Result<Arc<dyn TableProvider>> {+ let Some(location) = cmd.locations.first() else {+ return plan_err!("TestTableFactory requires at least one location");+ };+Ok(Arc::new(TestTableProvider {- url: cmd.location.to_string(),+ url: location.clone(),schema: Arc::clone(cmd.schema.inner()),}))}diff --git a/datafusion/expr/src/logical_plan/ddl.rs b/datafusion/expr/src/logical_plan/ddl.rsindex 1990a31edb..51d88e43c1 100644--- a/datafusion/expr/src/logical_plan/ddl.rs+++ b/datafusion/expr/src/logical_plan/ddl.rs@@ -211,8 +211,12 @@ pub struct CreateExternalTable {pub schema: DFSchemaRef,/// The table namepub name: TableReference,- /// The physical location- pub location: String,+ /// The physical locations of the table files.+ ///+ /// More than one location may be supplied (for example+ /// `CREATE EXTERNAL TABLE ... LOCATION ('a.parquet', 'b.parquet')`), in which+ /// case the files are read together as a single table.+ pub locations: Vec<String>,/// The file type of physical filepub file_type: String,/// Partition Columns@@ -266,7 +270,7 @@ impl CreateExternalTable {) -> CreateExternalTableBuilder {CreateExternalTableBuilder {name: name.into(),- location: location.into(),+ locations: vec![location.into()],file_type: file_type.into(),schema,table_partition_cols: vec![],@@ -289,7 +293,7 @@ impl CreateExternalTable {#[derive(Debug, Clone)]pub struct CreateExternalTableBuilder {name: TableReference,- location: String,+ locations: Vec<String>,file_type: String,schema: DFSchemaRef,table_partition_cols: Vec<String>,@@ -311,6 +315,16 @@ impl CreateExternalTableBuilder {self}+ /// Set the physical locations of the table files, replacing the single+ /// location supplied to [`CreateExternalTable::builder`].+ ///+ /// When more than one location is provided the files are read together as+ /// a single table.+ pub fn with_locations(mut self, locations: Vec<String>) -> Self {+ self.locations = locations;+ self+ }+/// Set the if_not_exists flagpub fn with_if_not_exists(mut self, if_not_exists: bool) -> Self {self.if_not_exists = if_not_exists;@@ -373,7 +387,7 @@ impl CreateExternalTableBuilder {CreateExternalTable {schema: self.schema,name: self.name,- location: self.location,+ locations: self.locations,file_type: self.file_type,table_partition_cols: self.table_partition_cols,if_not_exists: self.if_not_exists,@@ -394,7 +408,7 @@ impl Hash for CreateExternalTable {fn hash<H: Hasher>(&self, state: &mut H) {self.schema.hash(state);self.name.hash(state);- self.location.hash(state);+ self.locations.hash(state);self.file_type.hash(state);self.table_partition_cols.hash(state);self.if_not_exists.hash(state);@@ -413,8 +427,8 @@ impl PartialOrd for CreateExternalTable {struct ComparableCreateExternalTable<'a> {/// The table namepub name: &'a TableReference,- /// The physical location- pub location: &'a String,+ /// The physical locations+ pub locations: &'a Vec<String>,/// The file type of physical filepub file_type: &'a String,/// Partition Columns@@ -432,7 +446,7 @@ impl PartialOrd for CreateExternalTable {}let comparable_self = ComparableCreateExternalTable {name: &self.name,- location: &self.location,+ locations: &self.locations,file_type: &self.file_type,table_partition_cols: &self.table_partition_cols,if_not_exists: &self.if_not_exists,@@ -443,7 +457,7 @@ impl PartialOrd for CreateExternalTable {};let comparable_other = ComparableCreateExternalTable {name: &other.name,- location: &other.location,+ locations: &other.locations,file_type: &other.file_type,table_partition_cols: &other.table_partition_cols,if_not_exists: &other.if_not_exists,diff --git a/datafusion/ffi/src/table_provider_factory.rs b/datafusion/ffi/src/table_provider_factory.rsindex 466b56806d..b70e72f31a 100644--- a/datafusion/ffi/src/table_provider_factory.rs+++ b/datafusion/ffi/src/table_provider_factory.rs@@ -368,7 +368,7 @@ mod tests {let cmd = CreateExternalTable {schema: Schema::empty().to_dfschema_ref()?,name: TableReference::bare("test_table"),- location: "test".to_string(),+ locations: vec!["test".to_string()],file_type: "test".to_string(),table_partition_cols: vec![],if_not_exists: false,@@ -406,7 +406,7 @@ mod tests {let cmd = CreateExternalTable {schema: Schema::empty().to_dfschema_ref()?,name: TableReference::bare("cloned_test"),- location: "test".to_string(),+ locations: vec!["test".to_string()],file_type: "test".to_string(),table_partition_cols: vec![],if_not_exists: false,diff --git a/datafusion/ffi/tests/ffi_integration.rs b/datafusion/ffi/tests/ffi_integration.rsindex 6a6b6b3100..f1edc44730 100644--- a/datafusion/ffi/tests/ffi_integration.rs+++ b/datafusion/ffi/tests/ffi_integration.rs@@ -100,7 +100,7 @@ mod tests {let cmd = CreateExternalTable {schema: Schema::empty().to_dfschema_ref()?,name: TableReference::bare("cloned_test"),- location: "test".to_string(),+ locations: vec!["test".to_string()],file_type: "test".to_string(),table_partition_cols: vec![],if_not_exists: false,diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex fdb33c4fd2..205cf89abe 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -173,7 +173,8 @@ message EmptyRelationNode {message CreateExternalTableNode {reserved 1; // was string nameTableReference name = 9;- string location = 2;+ string location = 2; // deprecated; use repeated locations+ repeated string locations = 16;string file_type = 3;datafusion_common.DfSchema schema = 4;repeated string table_partition_cols = 5;diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 050d780d08..d23f8eee5f 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -3637,6 +3637,9 @@ impl serde::Serialize for CreateExternalTableNode {if !self.location.is_empty() {len += 1;}+ if !self.locations.is_empty() {+ len += 1;+ }if !self.file_type.is_empty() {len += 1;}@@ -3680,6 +3683,9 @@ impl serde::Serialize for CreateExternalTableNode {if !self.location.is_empty() {struct_ser.serialize_field("location", &self.location)?;}+ if !self.locations.is_empty() {+ struct_ser.serialize_field("locations", &self.locations)?;+ }if !self.file_type.is_empty() {struct_ser.serialize_field("fileType", &self.file_type)?;}@@ -3728,6 +3734,7 @@ impl<'de> serde::Deserialize<'de> for CreateExternalTableNode {const FIELDS: &[&str] = &["name","location",+ "locations","file_type","fileType","schema",@@ -3752,6 +3759,7 @@ impl<'de> serde::Deserialize<'de> for CreateExternalTableNode {enum GeneratedField {Name,Location,+ Locations,FileType,Schema,TablePartitionCols,@@ -3787,6 +3795,7 @@ impl<'de> serde::Deserialize<'de> for CreateExternalTableNode {match value {"name" => Ok(GeneratedField::Name),"location" => Ok(GeneratedField::Location),+ "locations" => Ok(GeneratedField::Locations),"fileType" | "file_type" => Ok(GeneratedField::FileType),"schema" => Ok(GeneratedField::Schema),"tablePartitionCols" | "table_partition_cols" => Ok(GeneratedField::TablePartitionCols),@@ -3820,6 +3829,7 @@ impl<'de> serde::Deserialize<'de> for CreateExternalTableNode {{let mut name__ = None;let mut location__ = None;+ let mut locations__ = None;let mut file_type__ = None;let mut schema__ = None;let mut table_partition_cols__ = None;@@ -3846,6 +3856,12 @@ impl<'de> serde::Deserialize<'de> for CreateExternalTableNode {}location__ = Some(map_.next_value()?);}+ GeneratedField::Locations => {+ if locations__.is_some() {+ return Err(serde::de::Error::duplicate_field("locations"));+ }+ locations__ = Some(map_.next_value()?);+ }GeneratedField::FileType => {if file_type__.is_some() {return Err(serde::de::Error::duplicate_field("fileType"));@@ -3927,6 +3943,7 @@ impl<'de> serde::Deserialize<'de> for CreateExternalTableNode {Ok(CreateExternalTableNode {name: name__,location: location__.unwrap_or_default(),+ locations: locations__.unwrap_or_default(),file_type: file_type__.unwrap_or_default(),schema: schema__,table_partition_cols: table_partition_cols__.unwrap_or_default(),diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 7fb3f15752..6baabbf37a 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -253,8 +253,11 @@ pub struct EmptyRelationNode {pub struct CreateExternalTableNode {#[prost(message, optional, tag = "9")]pub name: ::core::option::Option<TableReference>,+ /// deprecated; use repeated locations#[prost(string, tag = "2")]pub location: ::prost::alloc::string::String,+ #[prost(string, repeated, tag = "16")]+ pub locations: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,#[prost(string, tag = "3")]pub file_type: ::prost::alloc::string::String,#[prost(message, optional, tag = "4")]diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 3195b050b3..732676a3c0 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -799,6 +799,17 @@ impl AsLogicalPlan for LogicalPlanNode {column_defaults.insert(col_name.clone(), expr);}+ let locations = if !create_extern_table.locations.is_empty() {+ create_extern_table.locations.clone()+ } else if !create_extern_table.location.is_empty() {+ vec![create_extern_table.location.clone()]+ } else {+ return Err(proto_error(+ "CreateExternalTableNode requires at least one location",+ ));+ };+ let location = locations[0].clone();+Ok(LogicalPlan::Ddl(DdlStatement::CreateExternalTable(Box::new(CreateExternalTable::builder(@@ -806,10 +817,11 @@ impl AsLogicalPlan for LogicalPlanNode {create_extern_table.name.as_ref(),"CreateExternalTable",)?,- create_extern_table.location.clone(),+ location,create_extern_table.file_type.clone(),pb_schema.try_into()?,)+ .with_locations(locations).with_partition_cols(create_extern_table.table_partition_cols.clone(),)@@ -1804,7 +1816,7 @@ impl AsLogicalPlan for LogicalPlanNode {LogicalPlan::Ddl(DdlStatement::CreateExternalTable(ce)) => {let CreateExternalTable {name,- location,+ locations,file_type,schema: df_schema,table_partition_cols,@@ -1832,6 +1844,10 @@ impl AsLogicalPlan for LogicalPlanNode {converted_column_defaults.insert(col_name.clone(), serialize_expr(expr, extension_codec)?);}+ let (legacy_location, proto_locations) = match locations.as_slice() {+ [location] => (location.clone(), vec![]),+ _ => (String::new(), locations.clone()),+ };Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::CreateExternalTable(@@ -1839,7 +1855,8 @@ impl AsLogicalPlan for LogicalPlanNode {name: Some(protobuf::TableReference::from_proto(name.clone(),)),- location: location.clone(),+ location: legacy_location,+ locations: proto_locations,file_type: file_type.clone(),schema: Some(df_schema.try_into()?),table_partition_cols: table_partition_cols.clone(),diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex b3edc0f5ce..74f7253386 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -74,7 +74,7 @@ use datafusion_common::format::{};use datafusion_common::scalar::ScalarStructBuilder;use datafusion_common::{- DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, SplitPoint,+ Constraints, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, SplitPoint,TableReference, internal_datafusion_err, internal_err, not_impl_err, plan_err,};use datafusion_execution::TaskContext;@@ -116,7 +116,7 @@ use datafusion_proto::logical_plan::to_proto::serialize_expr;use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec, from_proto,};-use datafusion_proto::protobuf;+use datafusion_proto::{FromProto, protobuf};use crate::cases::{MyAggregateUDF, MyAggregateUdfNode, MyHigherOrderUDF, MyHigherOrderUdfNode,@@ -408,6 +408,131 @@ async fn roundtrip_custom_listing_tables() -> Result<()> {Ok(())}+#[tokio::test]+async fn roundtrip_create_external_table_multiple_locations() -> Result<()> {+ let ctx = SessionContext::new();++ // Planning a CREATE EXTERNAL TABLE does not read the referenced files, so+ // the paths need not exist. Multiple locations must survive the round-trip+ // through the `repeated locations` proto field.+ let query = "CREATE EXTERNAL TABLE t (a INTEGER, b INTEGER)+ STORED AS CSV+ LOCATION ('file_a.csv', 'file_b.csv')+ OPTIONS ('format.has_header' 'true')";++ let plan = ctx.state().create_logical_plan(query).await?;+ let bytes = logical_plan_to_bytes(&plan)?;+ let protobuf_plan = protobuf::LogicalPlanNode::decode(bytes.as_ref())+ .expect("failed to decode CreateExternalTable proto");+ #[cfg(feature = "json")]+ {+ let json = serde_json::to_string(&protobuf_plan).unwrap();+ assert!(!json.contains("\"location\":"));+ assert!(json.contains("\"locations\":[\"file_a.csv\",\"file_b.csv\"]"));+ }+ let Some(protobuf::logical_plan_node::LogicalPlanType::CreateExternalTable(+ create_external_table,+ )) = protobuf_plan.logical_plan_type+ else {+ panic!("expected a CreateExternalTable proto");+ };+ assert!(create_external_table.location.is_empty());+ assert_eq!(+ create_external_table.locations,+ vec!["file_a.csv".to_string(), "file_b.csv".to_string()]+ );++ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;+ assert_eq!(plan, logical_round_trip);++ let LogicalPlan::Ddl(datafusion_expr::DdlStatement::CreateExternalTable(rt)) =+ logical_round_trip+ else {+ panic!("expected a CreateExternalTable plan");+ };+ assert_eq!(+ rt.locations,+ vec!["file_a.csv".to_string(), "file_b.csv".to_string()]+ );++ Ok(())+}++#[tokio::test]+async fn roundtrip_create_external_table_single_location_legacy_field() -> Result<()> {+ let ctx = SessionContext::new();+ let query = "CREATE EXTERNAL TABLE t (a INTEGER)+ STORED AS CSV+ LOCATION 'file.csv'";++ let plan = ctx.state().create_logical_plan(query).await?;+ let bytes = logical_plan_to_bytes(&plan)?;+ let protobuf_plan = protobuf::LogicalPlanNode::decode(bytes.as_ref())+ .expect("failed to decode CreateExternalTable proto");+ #[cfg(feature = "json")]+ {+ let json = serde_json::to_string(&protobuf_plan).unwrap();+ assert!(json.contains("\"location\":\"file.csv\""));+ assert!(!json.contains("\"locations\""));+ }+ let Some(protobuf::logical_plan_node::LogicalPlanType::CreateExternalTable(+ create_external_table,+ )) = protobuf_plan.logical_plan_type+ else {+ panic!("expected a CreateExternalTable proto");+ };+ assert_eq!(create_external_table.location, "file.csv");+ assert!(create_external_table.locations.is_empty());++ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;+ assert_eq!(plan, logical_round_trip);++ Ok(())+}++#[tokio::test]+async fn roundtrip_create_external_table_legacy_location() -> Result<()> {+ let ctx = SessionContext::new();+ let schema = DFSchema::empty();+ let create_external_table = protobuf::CreateExternalTableNode {+ name: Some(protobuf::TableReference::from_proto(TableReference::bare(+ "t",+ ))),+ location: "legacy.csv".to_string(),+ locations: vec![],+ file_type: "CSV".to_string(),+ schema: Some((&schema).try_into()?),+ table_partition_cols: vec![],+ if_not_exists: false,+ or_replace: false,+ temporary: false,+ definition: String::new(),+ order_exprs: vec![],+ unbounded: false,+ options: HashMap::new(),+ constraints: Some(Constraints::default().into()),+ column_defaults: HashMap::new(),+ };+ let protobuf_plan = protobuf::LogicalPlanNode {+ logical_plan_type: Some(+ protobuf::logical_plan_node::LogicalPlanType::CreateExternalTable(+ create_external_table,+ ),+ ),+ };+ let bytes = protobuf_plan.encode_to_vec();++ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;+ let LogicalPlan::Ddl(datafusion_expr::DdlStatement::CreateExternalTable(rt)) =+ logical_round_trip+ else {+ panic!("expected a CreateExternalTable plan");+ };+ assert_eq!(rt.locations, vec!["legacy.csv".to_string()]);++ Ok(())+}+#[tokio::test]async fn roundtrip_logical_plan_aggregation_with_pk() -> Result<()> {let ctx = SessionContext::new();diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rsindex 0d7b7c63de..86a00ca767 100644--- a/datafusion/sql/src/parser.rs+++ b/datafusion/sql/src/parser.rs@@ -231,7 +231,7 @@ pub(crate) type LexOrdering = Vec<OrderByExpr>;/// [ PARTITIONED BY (<column_definition list> | <column list>) ]/// [ WITH ORDER (<ordered column list>)/// [ OPTIONS (<key_value_list>) ]-/// LOCATION <literal>+/// LOCATION <literal> | LOCATION (<literal>[, ...])////// <column_definition> := (<column_name> <data_type>, ...)///@@ -249,8 +249,8 @@ pub struct CreateExternalTable {pub columns: Vec<ColumnDef>,/// File type (Parquet, NDJSON, CSV, etc)pub file_type: String,- /// Path to file- pub location: String,+ /// Paths to files+ pub locations: Vec<String>,/// Partition Columnspub table_partition_cols: Vec<String>,/// Ordered expressions@@ -289,7 +289,23 @@ impl fmt::Display for CreateExternalTable {}write!(f, ") ")?;}- write!(f, "LOCATION {}", self.location)+ match self.locations.as_slice() {+ [location] => write!(+ f,+ "LOCATION {}",+ Value::SingleQuotedString(location.clone())+ ),+ locations => {+ write!(f, "LOCATION (")?;+ for (idx, location) in locations.iter().enumerate() {+ if idx > 0 {+ write!(f, ", ")?;+ }+ write!(f, "{}", Value::SingleQuotedString(location.clone()))?;+ }+ write!(f, ")")+ }+ }}}@@ -1097,7 +1113,7 @@ impl<'a> DFParser<'a> {#[derive(Default)]struct Builder {file_type: Option<String>,- location: Option<String>,+ locations: Option<Vec<String>>,table_partition_cols: Option<Vec<String>>,order_exprs: Vec<LexOrdering>,options: Option<Vec<(String, Value)>>,@@ -1121,8 +1137,8 @@ impl<'a> DFParser<'a> {builder.file_type = Some(self.parse_file_format()?);}Keyword::LOCATION => {- ensure_not_set(&builder.location, "LOCATION")?;- builder.location = Some(self.parser.parse_literal_string()?);+ ensure_not_set(&builder.locations, "LOCATION")?;+ builder.locations = Some(self.parse_locations()?);}Keyword::WITH => {if self.parser.parse_keyword(Keyword::ORDER) {@@ -1198,17 +1214,22 @@ impl<'a> DFParser<'a> {"Missing STORED AS clause in CREATE EXTERNAL TABLE statement".into(),));}- if builder.location.is_none() {+ if builder.locations.is_none() {return sql_err!(ParserError::ParserError("Missing LOCATION clause in CREATE EXTERNAL TABLE statement".into(),));}+ let locations = builder.locations.unwrap();+ if locations.is_empty() {+ return parser_err!("LOCATION requires at least one path");+ }+let create = CreateExternalTable {name: table_name,columns,file_type: builder.file_type.unwrap(),- location: builder.location.unwrap(),+ locations,table_partition_cols: builder.table_partition_cols.unwrap_or(vec![]),order_exprs: builder.order_exprs,if_not_exists,@@ -1221,6 +1242,29 @@ impl<'a> DFParser<'a> {Ok(Statement::CreateExternalTable(create))}+ /// Parses one or more external table locations.+ fn parse_locations(&mut self) -> Result<Vec<String>, DataFusionError> {+ if !self.parser.consume_token(&Token::LParen) {+ return Ok(vec![self.parser.parse_literal_string()?]);+ }++ let mut locations = vec![];+ loop {+ locations.push(self.parser.parse_literal_string()?);+ let comma = self.parser.consume_token(&Token::Comma);+ if self.parser.consume_token(&Token::RParen) {+ // Allow a trailing comma, even though it's not in standard+ break;+ } else if !comma {+ return self.expected(+ "',' or ')' after location definition",+ &self.parser.peek_token(),+ );+ }+ }+ Ok(locations)+ }+/// Parses the set of valid formatsfn parse_file_format(&mut self) -> Result<String, DataFusionError> {let token = self.parser.next_token();@@ -1309,17 +1353,23 @@ mod tests {}}- #[test]- fn create_external_table() -> Result<(), DataFusionError> {- // positive case- let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv'";- let display = None;- let name = ObjectName::from(vec![Ident::from("t")]);- let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![make_column_def("c1", DataType::Int(display))],+ fn make_create_external_table(location: &str) -> CreateExternalTable {+ make_create_external_table_with_locations(&[location])+ }++ fn make_create_external_table_with_locations(+ locations: &[&str],+ ) -> CreateExternalTable {+ let locations = locations+ .iter()+ .map(|location| location.to_string())+ .collect::<Vec<_>>();++ CreateExternalTable {+ name: ObjectName::from(vec![Ident::from("t")]),+ columns: vec![],file_type: "CSV".to_string(),- location: "foo.csv".into(),+ locations,table_partition_cols: vec![],order_exprs: vec![],if_not_exists: false,@@ -1328,24 +1378,59 @@ mod tests {unbounded: false,options: vec![],constraints: vec![],+ }+ }++ #[test]+ fn create_external_table() -> Result<(), DataFusionError> {+ // positive case+ let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv'";+ let display = None;+ let expected = Statement::CreateExternalTable(CreateExternalTable {+ columns: vec![make_column_def("c1", DataType::Int(display))],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;+ // positive case: literal comma remains part of a single path+ let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo,bar.csv'";+ let expected = Statement::CreateExternalTable(CreateExternalTable {+ columns: vec![make_column_def("c1", DataType::Int(display))],+ ..make_create_external_table("foo,bar.csv")+ });+ expect_parse_ok(sql, expected)?;++ // positive case: multiple locations use an explicit list+ let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION ('foo.csv', 'bar.csv')";+ let expected = Statement::CreateExternalTable(CreateExternalTable {+ columns: vec![make_column_def("c1", DataType::Int(display))],+ ..make_create_external_table_with_locations(&["foo.csv", "bar.csv"])+ });+ expect_parse_ok(sql, expected)?;++ assert_eq!(+ Statement::CreateExternalTable(make_create_external_table("foo.csv"))+ .to_string(),+ "CREATE EXTERNAL TABLE t STORED AS CSV LOCATION 'foo.csv'"+ );+ assert_eq!(+ Statement::CreateExternalTable(make_create_external_table_with_locations(&[+ "foo.csv", "bar.csv"+ ]))+ .to_string(),+ "CREATE EXTERNAL TABLE t STORED AS CSV LOCATION ('foo.csv', 'bar.csv')"+ );+ assert_eq!(+ Statement::CreateExternalTable(make_create_external_table("foo'bar.csv"))+ .to_string(),+ "CREATE EXTERNAL TABLE t STORED AS CSV LOCATION 'foo''bar.csv'"+ );+// positive case: leading spacelet sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' ";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(None))],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;@@ -1353,18 +1438,8 @@ mod tests {let sql ="CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' ;";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(None))],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;@@ -1372,21 +1447,12 @@ mod tests {let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS (format.delimiter '|')";let display = None;let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(display))],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,options: vec![("format.delimiter".into(),Value::SingleQuotedString("|".into()),)],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;@@ -1394,18 +1460,9 @@ mod tests {let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV PARTITIONED BY (p1, p2) LOCATION 'foo.csv'";let display = None;let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(display))],- file_type: "CSV".to_string(),- location: "foo.csv".into(),table_partition_cols: vec!["p1".to_string(), "p2".to_string()],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;@@ -1420,24 +1477,15 @@ mod tests {('format.compression' 'XZ')", "XZ"),("CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS('format.compression' 'ZSTD')", "ZSTD"),- ];+ ];for (sql, compression) in sqls {let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(display))],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,options: vec![("format.compression".into(),Value::SingleQuotedString(compression.into()),)],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;}@@ -1445,72 +1493,33 @@ mod tests {// positive case: it is ok for parquet files not to have columns specifiedlet sql = "CREATE EXTERNAL TABLE t STORED AS PARQUET LOCATION 'foo.parquet'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![],file_type: "PARQUET".to_string(),- location: "foo.parquet".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.parquet")});expect_parse_ok(sql, expected)?;// positive case: it is ok for parquet files to be other than upper caselet sql = "CREATE EXTERNAL TABLE t STORED AS parqueT LOCATION 'foo.parquet'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![],file_type: "PARQUET".to_string(),- location: "foo.parquet".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.parquet")});expect_parse_ok(sql, expected)?;// positive case: it is ok for avro files not to have columns specifiedlet sql = "CREATE EXTERNAL TABLE t STORED AS AVRO LOCATION 'foo.avro'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![],file_type: "AVRO".to_string(),- location: "foo.avro".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.avro")});expect_parse_ok(sql, expected)?;// positive case: it is ok for avro files not to have columns specifiedlet sql = "CREATE EXTERNAL TABLE IF NOT EXISTS t STORED AS PARQUET LOCATION 'foo.parquet'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![],file_type: "PARQUET".to_string(),- location: "foo.parquet".into(),- table_partition_cols: vec![],- order_exprs: vec![],if_not_exists: true,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.parquet")});expect_parse_ok(sql, expected)?;@@ -1518,39 +1527,21 @@ mod tests {let sql ="CREATE OR REPLACE EXTERNAL TABLE t STORED AS PARQUET LOCATION 'foo.parquet'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![],file_type: "PARQUET".to_string(),- location: "foo.parquet".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,or_replace: true,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.parquet")});expect_parse_ok(sql, expected)?;// positive case: column definition allowed in 'partition by' clauselet sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV PARTITIONED BY (p1 int) LOCATION 'foo.csv'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(None)),make_column_def("p1", DataType::Int(None)),],- file_type: "CSV".to_string(),- location: "foo.csv".into(),table_partition_cols: vec!["p1".to_string()],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;@@ -1572,39 +1563,21 @@ mod tests {let sql ="CREATE EXTERNAL TABLE t STORED AS x OPTIONS ('k1' 'v1') LOCATION 'blahblah'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![],file_type: "X".to_string(),- location: "blahblah".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,options: vec![("k1".into(), Value::SingleQuotedString("v1".into()))],- constraints: vec![],+ ..make_create_external_table("blahblah")});expect_parse_ok(sql, expected)?;// positive case: additional options (multiple entries) can be specifiedlet sql = "CREATE EXTERNAL TABLE t STORED AS x OPTIONS ('k1' 'v1', k2 v2) LOCATION 'blahblah'";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),- columns: vec![],file_type: "X".to_string(),- location: "blahblah".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,options: vec![("k1".into(), Value::SingleQuotedString("v1".into())),("k2".into(), Value::SingleQuotedString("v2".into())),],- constraints: vec![],+ ..make_create_external_table("blahblah")});expect_parse_ok(sql, expected)?;@@ -1633,11 +1606,7 @@ mod tests {];for (sql, (asc, nulls_first)) in sqls.iter().zip(expected) {let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(None))],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],order_exprs: vec![vec![OrderByExpr {expr: Identifier(Ident {value: "c1".to_owned(),@@ -1647,12 +1616,7 @@ mod tests {options: OrderByOptions { asc, nulls_first },with_fill: None,}]],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;}@@ -1661,14 +1625,10 @@ mod tests {let sql = "CREATE EXTERNAL TABLE t(c1 int, c2 int) STORED AS CSV WITH ORDER (c1 ASC, c2 DESC NULLS FIRST) LOCATION 'foo.csv'";let display = None;let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(display)),make_column_def("c2", DataType::Int(display)),],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],order_exprs: vec![vec![OrderByExpr {expr: Identifier(Ident {@@ -1695,12 +1655,7 @@ mod tests {with_fill: None,},]],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;@@ -1708,14 +1663,10 @@ mod tests {let sql = "CREATE EXTERNAL TABLE t(c1 int, c2 int) STORED AS CSV WITH ORDER (c1 - c2 ASC) LOCATION 'foo.csv'";let display = None;let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(display)),make_column_def("c2", DataType::Int(display)),],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],order_exprs: vec![vec![OrderByExpr {expr: Expr::BinaryOp {left: Box::new(Identifier(Ident {@@ -1736,12 +1687,7 @@ mod tests {},with_fill: None,}]],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")});expect_parse_ok(sql, expected)?;@@ -1758,13 +1704,11 @@ mod tests {'TRUNCATE' 'NO','format.has_header' 'true')";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(None)),make_column_def("c2", DataType::Float(ExactNumberInfo::None)),],file_type: "PARQUET".to_string(),- location: "foo.parquet".into(),table_partition_cols: vec!["c1".into()],order_exprs: vec![vec![OrderByExpr {expr: Expr::BinaryOp {@@ -1787,8 +1731,6 @@ mod tests {with_fill: None,}]],if_not_exists: true,- or_replace: false,- temporary: false,unbounded: true,options: vec![(@@ -1809,7 +1751,7 @@ mod tests {Value::SingleQuotedString("true".into()),),],- constraints: vec![],+ ..make_create_external_table("foo.parquet")});expect_parse_ok(sql, expected)?;@@ -1826,13 +1768,11 @@ mod tests {'TRUNCATE' 'NO','format.has_header' 'true')";let expected = Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(None)),make_column_def("c2", DataType::Float(ExactNumberInfo::None)),],file_type: "PARQUET".to_string(),- location: "foo.parquet".into(),table_partition_cols: vec!["c1".into()],order_exprs: vec![vec![OrderByExpr {expr: Expr::BinaryOp {@@ -1854,9 +1794,7 @@ mod tests {},with_fill: None,}]],- if_not_exists: false,or_replace: true,- temporary: false,unbounded: true,options: vec![(@@ -1877,7 +1815,7 @@ mod tests {Value::SingleQuotedString("true".into()),),],- constraints: vec![],+ ..make_create_external_table("foo.parquet")});expect_parse_ok(sql, expected)?;@@ -2152,21 +2090,10 @@ mod tests {options: vec![],}),{- let name = ObjectName::from(vec![Ident::from("t")]);let display = None;Statement::CreateExternalTable(CreateExternalTable {- name: name.clone(),columns: vec![make_column_def("c1", DataType::Int(display))],- file_type: "CSV".to_string(),- location: "foo.csv".into(),- table_partition_cols: vec![],- order_exprs: vec![],- if_not_exists: false,- or_replace: false,- temporary: false,- unbounded: false,- options: vec![],- constraints: vec![],+ ..make_create_external_table("foo.csv")})},{diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex 2b60d79b34..ae7579c8c4 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -1804,7 +1804,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {name,columns,file_type,- location,+ locations,table_partition_cols,if_not_exists,temporary,@@ -1853,9 +1853,17 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let name = self.object_name_to_table_reference(name)?;let constraints =self.new_constraint_from_table_constraints(&all_constraints, &df_schema)?;++ let Some(location) = locations.first().cloned() else {+ return plan_err!("CREATE EXTERNAL TABLE requires at least one location");+ };++ // Keep the existing single-location builder API: seed it with the first+ // location, then replace it with the complete list.Ok(LogicalPlan::Ddl(DdlStatement::CreateExternalTable(Box::new(PlanCreateExternalTable::builder(name, location, file_type, df_schema)+ .with_locations(locations).with_partition_cols(table_partition_cols).with_if_not_exists(if_not_exists).with_or_replace(or_replace)diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rsindex 88b7b43eb7..54480a4224 100644--- a/datafusion/sql/tests/sql_integration.rs+++ b/datafusion/sql/tests/sql_integration.rs@@ -2278,6 +2278,29 @@ fn create_external_table_csv() {);}+#[test]+fn create_external_table_multiple_locations() {+ let sql = "CREATE EXTERNAL TABLE t STORED AS CSV LOCATION ('foo.csv', 'bar.csv')";+ let plan = logical_plan(sql).unwrap();+ let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = plan else {+ panic!("expected a CreateExternalTable plan");+ };+ assert_eq!(+ cmd.locations,+ vec!["foo.csv".to_string(), "bar.csv".to_string()]+ );+}++#[test]+fn create_external_table_location_with_literal_comma() {+ let sql = "CREATE EXTERNAL TABLE t STORED AS CSV LOCATION 'foo,bar.csv'";+ let plan = logical_plan(sql).unwrap();+ let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = plan else {+ panic!("expected a CreateExternalTable plan");+ };+ assert_eq!(cmd.locations, vec!["foo,bar.csv".to_string()]);+}+#[test]fn create_external_table_with_pk() {let sql = "CREATE EXTERNAL TABLE t(c1 int, primary key(c1)) STORED AS CSV LOCATION 'foo.csv'";diff --git a/datafusion/sqllogictest/test_files/create_external_table.slt b/datafusion/sqllogictest/test_files/create_external_table.sltindex f56cff2a2a..1d339f4025 100644--- a/datafusion/sqllogictest/test_files/create_external_table.slt+++ b/datafusion/sqllogictest/test_files/create_external_table.slt@@ -303,3 +303,42 @@ statement error DataFusion error: SQL error: ParserError\("'IF NOT EXISTS' cannoCREATE OR REPLACE EXTERNAL TABLE IF NOT EXISTS t_conflict(c1 int)STORED AS CSVLOCATION 'foo.csv';++# Multiple listed locations are read together as a single table.+# Each partition-N.csv has 11 rows, so listing exactly two of them (rather than+# the whole directory) yields 22 rows.+statement ok+CREATE EXTERNAL TABLE multi_loc (c1 int, c2 bigint, c3 boolean)+STORED AS CSV+LOCATION ('../core/tests/data/partitioned_csv/partition-0.csv', '../core/tests/data/partitioned_csv/partition-1.csv')+OPTIONS ('format.has_header' 'false');++query I+SELECT count(*) FROM multi_loc;+----+22++statement ok+DROP TABLE multi_loc;++# Duplicate locations are rejected to avoid scanning the same data twice.+statement error Duplicate location+CREATE EXTERNAL TABLE multi_loc_duplicate (c1 int, c2 bigint, c3 boolean)+STORED AS CSV+LOCATION ('../core/tests/data/partitioned_csv/partition-0.csv', '../core/tests/data/partitioned_csv/partition-0.csv')+OPTIONS ('format.has_header' 'false');++# Whitespace around the list separators is ignored+statement ok+CREATE EXTERNAL TABLE multi_loc_ws (c1 int, c2 bigint, c3 boolean)+STORED AS CSV+LOCATION ( '../core/tests/data/partitioned_csv/partition-0.csv' , '../core/tests/data/partitioned_csv/partition-1.csv' )+OPTIONS ('format.has_header' 'false');++query I+SELECT count(*) FROM multi_loc_ws;+----+22++statement ok+DROP TABLE multi_loc_ws;diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex 12306b4529..77acaa4747 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -781,7 +781,7 @@ OPTIONS ('format.has_header' 'true');query TTTTSHOW CREATE TABLE abc;-----datafusion public abc CREATE EXTERNAL TABLE abc STORED AS CSV LOCATION ../../testing/data/csv/aggregate_test_100.csv+datafusion public abc CREATE EXTERNAL TABLE abc STORED AS CSV LOCATION '../../testing/data/csv/aggregate_test_100.csv'# show_external_create_table_with_orderstatement ok@@ -794,7 +794,7 @@ OPTIONS ('format.has_header' 'true');query TTTTSHOW CREATE TABLE abc_ordered;-----datafusion public abc_ordered CREATE EXTERNAL TABLE abc_ordered STORED AS CSV WITH ORDER (c1) LOCATION ../../testing/data/csv/aggregate_test_100.csv+datafusion public abc_ordered CREATE EXTERNAL TABLE abc_ordered STORED AS CSV WITH ORDER (c1) LOCATION '../../testing/data/csv/aggregate_test_100.csv'statement okDROP TABLE abc_ordered;@@ -810,7 +810,7 @@ OPTIONS ('format.has_header' 'true');query TTTTSHOW CREATE TABLE abc_multi_order;-----datafusion public abc_multi_order CREATE EXTERNAL TABLE abc_multi_order STORED AS CSV WITH ORDER (c1, c2 DESC) LOCATION ../../testing/data/csv/aggregate_test_100.csv+datafusion public abc_multi_order CREATE EXTERNAL TABLE abc_multi_order STORED AS CSV WITH ORDER (c1, c2 DESC) LOCATION '../../testing/data/csv/aggregate_test_100.csv'statement okDROP TABLE abc_multi_order;@@ -826,7 +826,7 @@ OPTIONS ('format.has_header' 'true');query TTTTSHOW CREATE TABLE abc_order_nulls;-----datafusion public abc_order_nulls CREATE EXTERNAL TABLE abc_order_nulls STORED AS CSV WITH ORDER (c1 NULLS LAST, c2 DESC NULLS FIRST) LOCATION ../../testing/data/csv/aggregate_test_100.csv+datafusion public abc_order_nulls CREATE EXTERNAL TABLE abc_order_nulls STORED AS CSV WITH ORDER (c1 NULLS LAST, c2 DESC NULLS FIRST) LOCATION '../../testing/data/csv/aggregate_test_100.csv'statement okDROP TABLE abc_order_nulls;diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex f52db8da93..28ceb27d63 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -150,6 +150,37 @@ This function was deprecated in DataFusion `46.0.0`.Use `datafusion_physical_plan::spill::SpillManager::spill_record_batch_by_size`instead.+### `CreateExternalTable` supports multiple locations++`CREATE EXTERNAL TABLE` now accepts multiple paths in a single `LOCATION`+clause, which are read together as one table:++```sql+CREATE EXTERNAL TABLE hits+STORED AS PARQUET+LOCATION ('file_1.parquet', 'file_2.parquet');+```++To support this, the `location` field of both+`datafusion_expr::CreateExternalTable` and+`datafusion_sql::parser::CreateExternalTable` changed from a `String` to a+`Vec<String>` named `locations`:++```rust+// Before (54.0.0)+let location: String = create_external_table.location;++// After (55.0.0)+let locations: Vec<String> = create_external_table.locations;+```++The `CreateExternalTable::builder(name, location, file_type, schema)`+constructor is unchanged and still takes a single location; use the new+`CreateExternalTableBuilder::with_locations(Vec<String>)` to set more than one.+All listed locations must resolve to the same schema and reside on the same+object store. A plain string literal remains a single location, so paths that+contain commas continue to work, for example `LOCATION 'path/with,comma.csv'`.+### Decimal scalar formatting uses human-readable valuesDecimal scalar literals in `EXPLAIN` output, expression display strings, anddiff --git a/docs/source/user-guide/sql/ddl.md b/docs/source/user-guide/sql/ddl.mdindex 3a5c934ae8..0d76775bcc 100644--- a/docs/source/user-guide/sql/ddl.md+++ b/docs/source/user-guide/sql/ddl.md@@ -82,6 +82,21 @@ For a comprehensive list of format-specific options that can be specified in thea path to a file or directory of partitioned files locally or on anobject store.+Multiple locations can be supplied as a parenthesized list of string literals,+in which case the files are read together as one table:++```sql+CREATE EXTERNAL TABLE hits+STORED AS PARQUET+LOCATION (+ 's3://clickhouse-public-datasets/hits_compatible/athena_partitioned/hits_1.parquet',+ 's3://clickhouse-public-datasets/hits_compatible/athena_partitioned/hits_2.parquet'+);+```++All listed locations must reside on the same object store and resolve to data+with the same schema.+### Example: ParquetParquet data sources can be registered by executing a `CREATE EXTERNAL TABLE` SQL statement such as the following. It is not necessary to