Apache DataFusion SQL Query Engine
Force `FileSource` to be constructed with a `Schema` (#18386)
Adrian Garcia Badaracco6ab4d216b768
diff --git a/datafusion-examples/examples/advanced_parquet_index.rs b/datafusion-examples/examples/advanced_parquet_index.rsindex 371c18de35..67bfc5b1bc 100644--- a/datafusion-examples/examples/advanced_parquet_index.rs+++ b/datafusion-examples/examples/advanced_parquet_index.rs@@ -491,19 +491,18 @@ impl TableProvider for IndexTableProvider {.with_file(indexed_file);let file_source = Arc::new(- ParquetSource::default()+ ParquetSource::new(schema.clone())// provide the predicate so the DataSourceExec can try and prune// row groups internally.with_predicate(predicate)// provide the factory to create parquet reader without re-reading metadata.with_parquet_file_reader_factory(Arc::new(reader_factory)),);- let file_scan_config =- FileScanConfigBuilder::new(object_store_url, schema, file_source)- .with_limit(limit)- .with_projection_indices(projection.cloned())- .with_file(partitioned_file)- .build();+ let file_scan_config = FileScanConfigBuilder::new(object_store_url, file_source)+ .with_limit(limit)+ .with_projection_indices(projection.cloned())+ .with_file(partitioned_file)+ .build();// Finally, put it all together into a DataSourceExecOk(DataSourceExec::from_data_source(file_scan_config))diff --git a/datafusion-examples/examples/csv_json_opener.rs b/datafusion-examples/examples/csv_json_opener.rsindex ef2a3eaca0..6d0e4f4a3d 100644--- a/datafusion-examples/examples/csv_json_opener.rs+++ b/datafusion-examples/examples/csv_json_opener.rs@@ -18,6 +18,7 @@use std::sync::Arc;use arrow::datatypes::{DataType, Field, Schema};+use datafusion::common::config::CsvOptions;use datafusion::{assert_batches_eq,datasource::{@@ -31,9 +32,7 @@ use datafusion::{test_util::aggr_test_schema,};-use datafusion::datasource::{- physical_plan::FileScanConfigBuilder, table_schema::TableSchema,-};+use datafusion::datasource::physical_plan::FileScanConfigBuilder;use futures::StreamExt;use object_store::{local::LocalFileSystem, memory::InMemory, ObjectStore};@@ -57,19 +56,25 @@ async fn csv_opener() -> Result<()> {let path = std::path::Path::new(&path).canonicalize()?;+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+let scan_config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),- Arc::clone(&schema),- Arc::new(CsvSource::default()),+ Arc::new(CsvSource::new(Arc::clone(&schema)).with_csv_options(options.clone())),).with_projection_indices(Some(vec![12, 0])).with_limit(Some(5)).with_file(PartitionedFile::new(path.display().to_string(), 10)).build();- let config = CsvSource::new(true, b',', b'"')+ let config = CsvSource::new(Arc::clone(&schema))+ .with_csv_options(options).with_comment(Some(b'#'))- .with_schema(TableSchema::from_file_schema(schema)).with_batch_size(8192).with_projection(&scan_config);@@ -125,8 +130,7 @@ async fn json_opener() -> Result<()> {let scan_config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),- schema,- Arc::new(JsonSource::default()),+ Arc::new(JsonSource::new(schema)),).with_projection_indices(Some(vec![1, 0])).with_limit(Some(5))diff --git a/datafusion-examples/examples/custom_file_format.rs b/datafusion-examples/examples/custom_file_format.rsindex 67fe642fd4..3505651eb1 100644--- a/datafusion-examples/examples/custom_file_format.rs+++ b/datafusion-examples/examples/custom_file_format.rs@@ -30,6 +30,7 @@ use datafusion::{FileFormat, FileFormatFactory,},physical_plan::{FileScanConfig, FileSinkConfig, FileSource},+ table_schema::TableSchema,MemTable,},error::Result,@@ -128,8 +129,8 @@ impl FileFormat for TSVFileFormat {.await}- fn file_source(&self) -> Arc<dyn FileSource> {- self.csv_file_format.file_source()+ fn file_source(&self, table_schema: TableSchema) -> Arc<dyn FileSource> {+ self.csv_file_format.file_source(table_schema)}}diff --git a/datafusion-examples/examples/default_column_values.rs b/datafusion-examples/examples/default_column_values.rsindex d3a7d2ec67..bfc60519f2 100644--- a/datafusion-examples/examples/default_column_values.rs+++ b/datafusion-examples/examples/default_column_values.rs@@ -235,7 +235,7 @@ impl TableProvider for DefaultValueTableProvider {&df_schema,)?;- let parquet_source = ParquetSource::default()+ let parquet_source = ParquetSource::new(schema.clone()).with_predicate(filter).with_pushdown_filters(true);@@ -257,7 +257,6 @@ impl TableProvider for DefaultValueTableProvider {let file_scan_config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("memory://")?,- self.schema.clone(),Arc::new(parquet_source),).with_projection_indices(projection.cloned())diff --git a/datafusion-examples/examples/parquet_embedded_index.rs b/datafusion-examples/examples/parquet_embedded_index.rsindex 3cbe189147..bc0e5a072c 100644--- a/datafusion-examples/examples/parquet_embedded_index.rs+++ b/datafusion-examples/examples/parquet_embedded_index.rs@@ -426,8 +426,10 @@ impl TableProvider for DistinctIndexTable {// Build ParquetSource to actually read the fileslet url = ObjectStoreUrl::parse("file://")?;- let source = Arc::new(ParquetSource::default().with_enable_page_index(true));- let mut builder = FileScanConfigBuilder::new(url, self.schema.clone(), source);+ let source = Arc::new(+ ParquetSource::new(self.schema.clone()).with_enable_page_index(true),+ );+ let mut builder = FileScanConfigBuilder::new(url, source);for file in files_to_scan {let path = self.dir.join(file);let len = std::fs::metadata(&path)?.len();diff --git a/datafusion-examples/examples/parquet_index.rs b/datafusion-examples/examples/parquet_index.rsindex a1dd1f1ffd..bc9e2a9226 100644--- a/datafusion-examples/examples/parquet_index.rs+++ b/datafusion-examples/examples/parquet_index.rs@@ -242,9 +242,10 @@ impl TableProvider for IndexTableProvider {let files = self.index.get_files(predicate.clone())?;let object_store_url = ObjectStoreUrl::parse("file://")?;- let source = Arc::new(ParquetSource::default().with_predicate(predicate));+ let source =+ Arc::new(ParquetSource::new(self.schema()).with_predicate(predicate));let mut file_scan_config_builder =- FileScanConfigBuilder::new(object_store_url, self.schema(), source)+ FileScanConfigBuilder::new(object_store_url, source).with_projection_indices(projection.cloned()).with_limit(limit);diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 95f9523d44..33d5c86bf8 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -34,7 +34,7 @@ use datafusion_datasource::schema_adapter::{DefaultSchemaAdapterFactory, SchemaAdapter, SchemaAdapterFactory,};use datafusion_datasource::{- compute_all_files_statistics, ListingTableUrl, PartitionedFile,+ compute_all_files_statistics, ListingTableUrl, PartitionedFile, TableSchema,};use datafusion_execution::cache::cache_manager::FileStatisticsCache;use datafusion_execution::cache::cache_unit::DefaultFileStatisticsCache;@@ -338,7 +338,16 @@ impl ListingTable {fn create_file_source_with_schema_adapter(&self,) -> datafusion_common::Result<Arc<dyn FileSource>> {- let mut source = self.options.format.file_source();+ let table_schema = TableSchema::new(+ Arc::clone(&self.file_schema),+ self.options+ .table_partition_cols+ .iter()+ .map(|(col, field)| Arc::new(Field::new(col, field.clone(), false)))+ .collect(),+ );++ let mut source = self.options.format.file_source(table_schema);// Apply schema adapter to source if available//// The source will use this SchemaAdapter to adapt data batches as they flow up the plan.@@ -418,7 +427,7 @@ impl TableProvider for ListingTable {.options.table_partition_cols.iter()- .map(|col| Ok(self.table_schema.field_with_name(&col.0)?.clone()))+ .map(|col| Ok(Arc::new(self.table_schema.field_with_name(&col.0)?.clone()))).collect::<datafusion_common::Result<Vec<_>>>()?;let table_partition_col_names = table_partition_cols@@ -491,20 +500,15 @@ impl TableProvider for ListingTable {.format.create_physical_plan(state,- FileScanConfigBuilder::new(- object_store_url,- Arc::clone(&self.file_schema),- file_source,- )- .with_file_groups(partitioned_file_lists)- .with_constraints(self.constraints.clone())- .with_statistics(statistics)- .with_projection_indices(projection)- .with_limit(limit)- .with_output_ordering(output_ordering)- .with_table_partition_cols(table_partition_cols)- .with_expr_adapter(self.expr_adapter_factory.clone())- .build(),+ FileScanConfigBuilder::new(object_store_url, file_source)+ .with_file_groups(partitioned_file_lists)+ .with_constraints(self.constraints.clone())+ .with_statistics(statistics)+ .with_projection_indices(projection)+ .with_limit(limit)+ .with_output_ordering(output_ordering)+ .with_expr_adapter(self.expr_adapter_factory.clone())+ .build(),).await?;diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rsindex 4881783eeb..7c55d452c4 100644--- a/datafusion/core/src/datasource/file_format/mod.rs+++ b/datafusion/core/src/datasource/file_format/mod.rs@@ -40,6 +40,7 @@ pub(crate) mod test_util {use datafusion_catalog::Session;use datafusion_common::Result;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;+ use datafusion_datasource::TableSchema;use datafusion_datasource::{file_format::FileFormat, PartitionedFile};use datafusion_execution::object_store::ObjectStoreUrl;use std::sync::Arc;@@ -66,6 +67,8 @@ pub(crate) mod test_util {.await?};+ let table_schema = TableSchema::new(file_schema.clone(), vec![]);+let statistics = format.infer_stats(state, &store, file_schema.clone(), &meta).await?;@@ -85,8 +88,7 @@ pub(crate) mod test_util {state,FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),- file_schema,- format.file_source(),+ format.file_source(table_schema),).with_file_groups(file_groups).with_statistics(statistics)diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rsindex 37b9663111..620e389a0f 100644--- a/datafusion/core/src/datasource/mod.rs+++ b/datafusion/core/src/datasource/mod.rs@@ -124,16 +124,13 @@ mod tests {let f2 = Field::new("extra_column", DataType::Utf8, true);let schema = Arc::new(Schema::new(vec![f1.clone(), f2.clone()]));- let source = ParquetSource::default()+ let source = ParquetSource::new(Arc::clone(&schema)).with_schema_adapter_factory(Arc::new(TestSchemaAdapterFactory {})).unwrap();- let base_conf = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- schema,- source,- )- .with_file(partitioned_file)- .build();+ let base_conf =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file(partitioned_file)+ .build();let parquet_exec = DataSourceExec::from_data_source(base_conf);diff --git a/datafusion/core/src/datasource/physical_plan/avro.rs b/datafusion/core/src/datasource/physical_plan/avro.rsindex 9068c97581..1cf8c573ac 100644--- a/datafusion/core/src/datasource/physical_plan/avro.rs+++ b/datafusion/core/src/datasource/physical_plan/avro.rs@@ -34,7 +34,7 @@ mod tests {use datafusion_common::{test_util, Result, ScalarValue};use datafusion_datasource::file_format::FileFormat;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;- use datafusion_datasource::PartitionedFile;+ use datafusion_datasource::{PartitionedFile, TableSchema};use datafusion_datasource_avro::source::AvroSource;use datafusion_datasource_avro::AvroFormat;use datafusion_execution::object_store::ObjectStoreUrl;@@ -81,15 +81,11 @@ mod tests {.infer_schema(&state, &store, std::slice::from_ref(&meta)).await?;- let source = Arc::new(AvroSource::new());- let conf = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- file_schema,- source,- )- .with_file(meta.into())- .with_projection_indices(Some(vec![0, 1, 2]))- .build();+ let source = Arc::new(AvroSource::new(Arc::clone(&file_schema)));+ let conf = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file(meta.into())+ .with_projection_indices(Some(vec![0, 1, 2]))+ .build();let source_exec = DataSourceExec::from_data_source(conf);assert_eq!(@@ -157,8 +153,8 @@ mod tests {// Include the missing column in the projectionlet projection = Some(vec![0, 1, 2, actual_schema.fields().len()]);- let source = Arc::new(AvroSource::new());- let conf = FileScanConfigBuilder::new(object_store_url, file_schema, source)+ let source = Arc::new(AvroSource::new(Arc::clone(&file_schema)));+ let conf = FileScanConfigBuilder::new(object_store_url, source).with_file(meta.into()).with_projection_indices(projection).build();@@ -227,13 +223,16 @@ mod tests {partitioned_file.partition_values = vec![ScalarValue::from("2021-10-26")];let projection = Some(vec![0, 1, file_schema.fields().len(), 2]);- let source = Arc::new(AvroSource::new());- let conf = FileScanConfigBuilder::new(object_store_url, file_schema, source)+ let table_schema = TableSchema::new(+ file_schema.clone(),+ vec![Arc::new(Field::new("date", DataType::Utf8, false))],+ );+ let source = Arc::new(AvroSource::new(table_schema.clone()));+ let conf = FileScanConfigBuilder::new(object_store_url, source)// select specific columns of the files as well as the partitioning// column which is supposed to be the last column in the table schema..with_projection_indices(projection).with_file(partitioned_file)- .with_table_partition_cols(vec![Field::new("date", DataType::Utf8, false)]).build();let source_exec = DataSourceExec::from_data_source(conf);diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rsindex 4f46a57d8b..ac5df24d49 100644--- a/datafusion/core/src/datasource/physical_plan/csv.rs+++ b/datafusion/core/src/datasource/physical_plan/csv.rs@@ -29,12 +29,14 @@ mod tests {use std::io::Write;use std::sync::Arc;+ use datafusion_datasource::TableSchema;use datafusion_datasource_csv::CsvFormat;use object_store::ObjectStore;use crate::prelude::CsvReadOptions;use crate::prelude::SessionContext;use crate::test::partitioned_file_groups;+ use datafusion_common::config::CsvOptions;use datafusion_common::test_util::arrow_test_data;use datafusion_common::test_util::batches_to_string;use datafusion_common::{assert_batches_eq, Result};@@ -94,6 +96,8 @@ mod tests {async fn csv_exec_with_projection(file_compression_type: FileCompressionType,) -> Result<()> {+ use datafusion_datasource::TableSchema;+let session_ctx = SessionContext::new();let task_ctx = session_ctx.task_ctx();let file_schema = aggr_test_schema();@@ -110,16 +114,21 @@ mod tests {tmp_dir.path(),)?;- let source = Arc::new(CsvSource::new(true, b',', b'"'));- let config = FileScanConfigBuilder::from(partitioned_csv_config(- file_schema,- file_groups,- source,- ))- .with_file_compression_type(file_compression_type)- .with_newlines_in_values(false)- .with_projection_indices(Some(vec![0, 2, 4]))- .build();+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));+ let source =+ Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));+ let config =+ FileScanConfigBuilder::from(partitioned_csv_config(file_groups, source))+ .with_file_compression_type(file_compression_type)+ .with_newlines_in_values(false)+ .with_projection_indices(Some(vec![0, 2, 4]))+ .build();assert_eq!(13, config.file_schema().fields().len());let csv = DataSourceExec::from_data_source(config);@@ -158,6 +167,8 @@ mod tests {async fn csv_exec_with_mixed_order_projection(file_compression_type: FileCompressionType,) -> Result<()> {+ use datafusion_datasource::TableSchema;+let cfg = SessionConfig::new().set_str("datafusion.catalog.has_header", "true");let session_ctx = SessionContext::new_with_config(cfg);let task_ctx = session_ctx.task_ctx();@@ -175,16 +186,21 @@ mod tests {tmp_dir.path(),)?;- let source = Arc::new(CsvSource::new(true, b',', b'"'));- let config = FileScanConfigBuilder::from(partitioned_csv_config(- file_schema,- file_groups,- source,- ))- .with_newlines_in_values(false)- .with_file_compression_type(file_compression_type.to_owned())- .with_projection_indices(Some(vec![4, 0, 2]))- .build();+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));+ let source =+ Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));+ let config =+ FileScanConfigBuilder::from(partitioned_csv_config(file_groups, source))+ .with_newlines_in_values(false)+ .with_file_compression_type(file_compression_type.to_owned())+ .with_projection_indices(Some(vec![4, 0, 2]))+ .build();assert_eq!(13, config.file_schema().fields().len());let csv = DataSourceExec::from_data_source(config);assert_eq!(3, csv.schema().fields().len());@@ -221,6 +237,7 @@ mod tests {async fn csv_exec_with_limit(file_compression_type: FileCompressionType,) -> Result<()> {+ use datafusion_datasource::TableSchema;use futures::StreamExt;let cfg = SessionConfig::new().set_str("datafusion.catalog.has_header", "true");@@ -240,16 +257,21 @@ mod tests {tmp_dir.path(),)?;- let source = Arc::new(CsvSource::new(true, b',', b'"'));- let config = FileScanConfigBuilder::from(partitioned_csv_config(- file_schema,- file_groups,- source,- ))- .with_newlines_in_values(false)- .with_file_compression_type(file_compression_type.to_owned())- .with_limit(Some(5))- .build();+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));+ let source =+ Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));+ let config =+ FileScanConfigBuilder::from(partitioned_csv_config(file_groups, source))+ .with_newlines_in_values(false)+ .with_file_compression_type(file_compression_type.to_owned())+ .with_limit(Some(5))+ .build();assert_eq!(13, config.file_schema().fields().len());let csv = DataSourceExec::from_data_source(config);assert_eq!(13, csv.schema().fields().len());@@ -287,6 +309,8 @@ mod tests {async fn csv_exec_with_missing_column(file_compression_type: FileCompressionType,) -> Result<()> {+ use datafusion_datasource::TableSchema;+let session_ctx = SessionContext::new();let task_ctx = session_ctx.task_ctx();let file_schema = aggr_test_schema_with_missing_col();@@ -303,16 +327,21 @@ mod tests {tmp_dir.path(),)?;- let source = Arc::new(CsvSource::new(true, b',', b'"'));- let config = FileScanConfigBuilder::from(partitioned_csv_config(- file_schema,- file_groups,- source,- ))- .with_newlines_in_values(false)- .with_file_compression_type(file_compression_type.to_owned())- .with_limit(Some(5))- .build();+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));+ let source =+ Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));+ let config =+ FileScanConfigBuilder::from(partitioned_csv_config(file_groups, source))+ .with_newlines_in_values(false)+ .with_file_compression_type(file_compression_type.to_owned())+ .with_limit(Some(5))+ .build();assert_eq!(14, config.file_schema().fields().len());let csv = DataSourceExec::from_data_source(config);assert_eq!(14, csv.schema().fields().len());@@ -341,6 +370,7 @@ mod tests {file_compression_type: FileCompressionType,) -> Result<()> {use datafusion_common::ScalarValue;+ use datafusion_datasource::TableSchema;let session_ctx = SessionContext::new();let task_ctx = session_ctx.task_ctx();@@ -362,19 +392,26 @@ mod tests {let num_file_schema_fields = file_schema.fields().len();- let source = Arc::new(CsvSource::new(true, b',', b'"'));- let config = FileScanConfigBuilder::from(partitioned_csv_config(- file_schema,- file_groups,- source,- ))- .with_newlines_in_values(false)- .with_file_compression_type(file_compression_type.to_owned())- .with_table_partition_cols(vec![Field::new("date", DataType::Utf8, false)])- // We should be able to project on the partition column- // Which is supposed to be after the file fields- .with_projection_indices(Some(vec![0, num_file_schema_fields]))- .build();+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::new(+ Arc::clone(&file_schema),+ vec![Arc::new(Field::new("date", DataType::Utf8, false))],+ );+ let source =+ Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));+ let config =+ FileScanConfigBuilder::from(partitioned_csv_config(file_groups, source))+ .with_newlines_in_values(false)+ .with_file_compression_type(file_compression_type.to_owned())+ // We should be able to project on the partition column+ // Which is supposed to be after the file fields+ .with_projection_indices(Some(vec![0, num_file_schema_fields]))+ .build();// we don't have `/date=xx/` in the path but that is ok because// partitions are resolved during scan anyway@@ -463,15 +500,20 @@ mod tests {).unwrap();- let source = Arc::new(CsvSource::new(true, b',', b'"'));- let config = FileScanConfigBuilder::from(partitioned_csv_config(- file_schema,- file_groups,- source,- ))- .with_newlines_in_values(false)- .with_file_compression_type(file_compression_type.to_owned())- .build();+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));+ let source =+ Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));+ let config =+ FileScanConfigBuilder::from(partitioned_csv_config(file_groups, source))+ .with_newlines_in_values(false)+ .with_file_compression_type(file_compression_type.to_owned())+ .build();let csv = DataSourceExec::from_data_source(config);let it = csv.execute(0, task_ctx).unwrap();diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rsindex f7d5c710bf..de7e87d25c 100644--- a/datafusion/core/src/datasource/physical_plan/json.rs+++ b/datafusion/core/src/datasource/physical_plan/json.rs@@ -176,8 +176,8 @@ mod tests {let (object_store_url, file_groups, file_schema) =prepare_store(&state, file_compression_type.to_owned(), tmp_dir.path()).await;- let source = Arc::new(JsonSource::new());- let conf = FileScanConfigBuilder::new(object_store_url, file_schema, source)+ let source = Arc::new(JsonSource::new(Arc::clone(&file_schema)));+ let conf = FileScanConfigBuilder::new(object_store_url, source).with_file_groups(file_groups).with_limit(Some(3)).with_file_compression_type(file_compression_type.to_owned())@@ -251,8 +251,8 @@ mod tests {let file_schema = Arc::new(builder.finish());let missing_field_idx = file_schema.fields.len() - 1;- let source = Arc::new(JsonSource::new());- let conf = FileScanConfigBuilder::new(object_store_url, file_schema, source)+ let source = Arc::new(JsonSource::new(Arc::clone(&file_schema)));+ let conf = FileScanConfigBuilder::new(object_store_url, source).with_file_groups(file_groups).with_limit(Some(3)).with_file_compression_type(file_compression_type.to_owned())@@ -294,8 +294,8 @@ mod tests {let (object_store_url, file_groups, file_schema) =prepare_store(&state, file_compression_type.to_owned(), tmp_dir.path()).await;- let source = Arc::new(JsonSource::new());- let conf = FileScanConfigBuilder::new(object_store_url, file_schema, source)+ let source = Arc::new(JsonSource::new(Arc::clone(&file_schema)));+ let conf = FileScanConfigBuilder::new(object_store_url, source).with_file_groups(file_groups).with_projection_indices(Some(vec![0, 2])).with_file_compression_type(file_compression_type.to_owned())@@ -342,8 +342,8 @@ mod tests {let (object_store_url, file_groups, file_schema) =prepare_store(&state, file_compression_type.to_owned(), tmp_dir.path()).await;- let source = Arc::new(JsonSource::new());- let conf = FileScanConfigBuilder::new(object_store_url, file_schema, source)+ let source = Arc::new(JsonSource::new(Arc::clone(&file_schema)));+ let conf = FileScanConfigBuilder::new(object_store_url, source).with_file_groups(file_groups).with_projection_indices(Some(vec![3, 0, 2])).with_file_compression_type(file_compression_type.to_owned())diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rsindex 0ffb252a66..b27dcf56e3 100644--- a/datafusion/core/src/datasource/physical_plan/parquet.rs+++ b/datafusion/core/src/datasource/physical_plan/parquet.rs@@ -161,7 +161,7 @@ mod tests {.as_ref().map(|p| logical2physical(p, &table_schema));- let mut source = ParquetSource::default();+ let mut source = ParquetSource::new(table_schema);if let Some(predicate) = predicate {source = source.with_predicate(predicate);}@@ -186,23 +186,19 @@ mod tests {source = source.with_bloom_filter_on_read(false);}- source.with_schema(TableSchema::new(Arc::clone(&table_schema), vec![]))+ Arc::new(source)}fn build_parquet_exec(&self,- file_schema: SchemaRef,file_group: FileGroup,source: Arc<dyn FileSource>,) -> Arc<DataSourceExec> {- let base_config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- file_schema,- source,- )- .with_file_group(file_group)- .with_projection_indices(self.projection.clone())- .build();+ let base_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file_group(file_group)+ .with_projection_indices(self.projection.clone())+ .build();DataSourceExec::from_data_source(base_config)}@@ -231,11 +227,8 @@ mod tests {// build a ParquetExec to return the resultslet parquet_source = self.build_file_source(Arc::clone(table_schema));- let parquet_exec = self.build_parquet_exec(- Arc::clone(table_schema),- file_group.clone(),- Arc::clone(&parquet_source),- );+ let parquet_exec =+ self.build_parquet_exec(file_group.clone(), Arc::clone(&parquet_source));let analyze_exec = Arc::new(AnalyzeExec::new(false,@@ -243,7 +236,6 @@ mod tests {vec![MetricType::SUMMARY, MetricType::DEV],// use a new ParquetSource to avoid sharing execution metricsself.build_parquet_exec(- Arc::clone(table_schema),file_group.clone(),self.build_file_source(Arc::clone(table_schema)),),@@ -1550,8 +1542,7 @@ mod tests {) -> Result<()> {let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),- file_schema,- Arc::new(ParquetSource::default()),+ Arc::new(ParquetSource::new(file_schema)),).with_file_groups(file_groups).build();@@ -1653,23 +1644,26 @@ mod tests {),]);- let source = Arc::new(ParquetSource::default());- let config = FileScanConfigBuilder::new(object_store_url, schema.clone(), source)- .with_file(partitioned_file)- // file has 10 cols so index 12 should be month and 13 should be day- .with_projection_indices(Some(vec![0, 1, 2, 12, 13]))- .with_table_partition_cols(vec![- Field::new("year", DataType::Utf8, false),- Field::new("month", DataType::UInt8, false),- Field::new(+ let table_schema = TableSchema::new(+ Arc::clone(&schema),+ vec![+ Arc::new(Field::new("year", DataType::Utf8, false)),+ Arc::new(Field::new("month", DataType::UInt8, false)),+ Arc::new(Field::new("day",DataType::Dictionary(Box::new(DataType::UInt16),Box::new(DataType::Utf8),),false,- ),- ])+ )),+ ],+ );+ let source = Arc::new(ParquetSource::new(table_schema.clone()));+ let config = FileScanConfigBuilder::new(object_store_url, source)+ .with_file(partitioned_file)+ // file has 10 cols so index 12 should be month and 13 should be day+ .with_projection_indices(Some(vec![0, 1, 2, 12, 13])).build();let parquet_exec = DataSourceExec::from_data_source(config);@@ -1731,8 +1725,7 @@ mod tests {let file_schema = Arc::new(Schema::empty());let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),- file_schema,- Arc::new(ParquetSource::default()),+ Arc::new(ParquetSource::new(file_schema)),).with_file(partitioned_file).build();@@ -2279,11 +2272,11 @@ mod tests {let size_hint_calls = reader_factory.metadata_size_hint_calls.clone();let source = Arc::new(- ParquetSource::default()+ ParquetSource::new(Arc::clone(&schema)).with_parquet_file_reader_factory(reader_factory).with_metadata_size_hint(456),);- let config = FileScanConfigBuilder::new(store_url, schema, source)+ let config = FileScanConfigBuilder::new(store_url, source).with_file(PartitionedFile {object_meta: ObjectMeta {diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rsindex 68f83e7f1f..bbc85af7d8 100644--- a/datafusion/core/src/test/mod.rs+++ b/datafusion/core/src/test/mod.rs@@ -35,12 +35,15 @@ use crate::error::Result;use crate::logical_expr::LogicalPlan;use crate::test_util::{aggr_test_schema, arrow_test_data};+use datafusion_common::config::CsvOptions;+use arrow::array::{self, Array, ArrayRef, Decimal128Builder, Int32Array};use arrow::datatypes::{DataType, Field, Schema};use arrow::record_batch::RecordBatch;#[cfg(feature = "compression")]use datafusion_common::DataFusionError;use datafusion_datasource::source::DataSourceExec;+use datafusion_datasource::TableSchema;#[cfg(feature = "compression")]use bzip2::write::BzEncoder;@@ -92,11 +95,17 @@ pub fn scan_partitioned_csv(FileCompressionType::UNCOMPRESSED,work_dir,)?;- let source = Arc::new(CsvSource::new(true, b'"', b'"'));- let config =- FileScanConfigBuilder::from(partitioned_csv_config(schema, file_groups, source))- .with_file_compression_type(FileCompressionType::UNCOMPRESSED)- .build();+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::from_file_schema(schema);+ let source = Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));+ let config = FileScanConfigBuilder::from(partitioned_csv_config(file_groups, source))+ .with_file_compression_type(FileCompressionType::UNCOMPRESSED)+ .build();Ok(DataSourceExec::from_data_source(config))}diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rsindex 203d9e97d2..b5213cee3f 100644--- a/datafusion/core/src/test_util/parquet.rs+++ b/datafusion/core/src/test_util/parquet.rs@@ -37,10 +37,8 @@ use crate::physical_plan::metrics::MetricsSet;use crate::physical_plan::ExecutionPlan;use crate::prelude::{Expr, SessionConfig, SessionContext};-use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;-use datafusion_datasource::TableSchema;use object_store::path::Path;use object_store::ObjectMeta;use parquet::arrow::ArrowWriter;@@ -157,20 +155,21 @@ impl TestParquetFile {maybe_filter: Option<Expr>,) -> Result<Arc<dyn ExecutionPlan>> {let parquet_options = ctx.copied_table_options().parquet;- let source = Arc::new(ParquetSource::new(parquet_options.clone()));- let scan_config_builder = FileScanConfigBuilder::new(- self.object_store_url.clone(),- Arc::clone(&self.schema),- source,- )- .with_file(PartitionedFile {- object_meta: self.object_meta.clone(),- partition_values: vec![],- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- });+ let source = Arc::new(+ ParquetSource::new(Arc::clone(&self.schema))+ .with_table_parquet_options(parquet_options.clone()),+ );+ let scan_config_builder =+ FileScanConfigBuilder::new(self.object_store_url.clone(), source).with_file(+ PartitionedFile {+ object_meta: self.object_meta.clone(),+ partition_values: vec![],+ range: None,+ statistics: None,+ extensions: None,+ metadata_size_hint: None,+ },+ );let df_schema = Arc::clone(&self.schema).to_dfschema_ref()?;@@ -184,10 +183,10 @@ impl TestParquetFile {create_physical_expr(&filter, &df_schema, &ExecutionProps::default())?;let source = Arc::new(- ParquetSource::new(parquet_options)+ ParquetSource::new(Arc::clone(&self.schema))+ .with_table_parquet_options(parquet_options).with_predicate(Arc::clone(&physical_filter_expr)),- )- .with_schema(TableSchema::from_file_schema(Arc::clone(&self.schema)));+ );let config = scan_config_builder.with_source(source).build();let parquet_exec = DataSourceExec::from_data_source(config);diff --git a/datafusion/core/tests/fuzz_cases/pruning.rs b/datafusion/core/tests/fuzz_cases/pruning.rsindex f8bd4dbc1a..51ec8f03e5 100644--- a/datafusion/core/tests/fuzz_cases/pruning.rs+++ b/datafusion/core/tests/fuzz_cases/pruning.rs@@ -276,13 +276,12 @@ async fn execute_with_predicate(ctx: &SessionContext,) -> Vec<String> {let parquet_source = if prune_stats {- ParquetSource::default().with_predicate(predicate.clone())+ ParquetSource::new(schema.clone()).with_predicate(predicate.clone())} else {- ParquetSource::default()+ ParquetSource::new(schema.clone())};let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("memory://").unwrap(),- schema.clone(),Arc::new(parquet_source),).with_file_group(diff --git a/datafusion/core/tests/parquet/custom_reader.rs b/datafusion/core/tests/parquet/custom_reader.rsindex 3a1f066562..0a147d15a6 100644--- a/datafusion/core/tests/parquet/custom_reader.rs+++ b/datafusion/core/tests/parquet/custom_reader.rs@@ -80,7 +80,7 @@ async fn route_data_access_ops_to_parquet_file_reader_factory() {.collect();let source = Arc::new(- ParquetSource::default()+ ParquetSource::new(file_schema.clone())// prepare the scan.with_parquet_file_reader_factory(Arc::new(InMemoryParquetFileReaderFactory(Arc::clone(&in_memory_object_store)),@@ -89,7 +89,6 @@ async fn route_data_access_ops_to_parquet_file_reader_factory() {let base_config = FileScanConfigBuilder::new(// just any url that doesn't point to in memory object storeObjectStoreUrl::local_filesystem(),- file_schema,source,).with_file_group(file_group)diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rsindex 5135f95685..b35cb6e09c 100644--- a/datafusion/core/tests/parquet/external_access_plan.rs+++ b/datafusion/core/tests/parquet/external_access_plan.rs@@ -355,11 +355,11 @@ impl TestFull {let source = if let Some(predicate) = predicate {let df_schema = DFSchema::try_from(schema.clone())?;let predicate = ctx.create_physical_expr(predicate, &df_schema)?;- Arc::new(ParquetSource::default().with_predicate(predicate))+ Arc::new(ParquetSource::new(schema.clone()).with_predicate(predicate))} else {- Arc::new(ParquetSource::default())+ Arc::new(ParquetSource::new(schema.clone()))};- let config = FileScanConfigBuilder::new(object_store_url, schema.clone(), source)+ let config = FileScanConfigBuilder::new(object_store_url, source).with_file(partitioned_file).build();diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rsindex 27bee10234..fb2a196b0a 100644--- a/datafusion/core/tests/parquet/page_pruning.rs+++ b/datafusion/core/tests/parquet/page_pruning.rs@@ -81,12 +81,12 @@ async fn get_parquet_exec(let predicate = create_physical_expr(&filter, &df_schema, &execution_props).unwrap();let source = Arc::new(- ParquetSource::default()+ ParquetSource::new(schema.clone()).with_predicate(predicate).with_enable_page_index(true).with_pushdown_filters(pushdown_filters),);- let base_config = FileScanConfigBuilder::new(object_store_url, schema, source)+ let base_config = FileScanConfigBuilder::new(object_store_url, source).with_file(partitioned_file).build();diff --git a/datafusion/core/tests/parquet/schema_adapter.rs b/datafusion/core/tests/parquet/schema_adapter.rsindex 40fc6176e2..0e76d626aa 100644--- a/datafusion/core/tests/parquet/schema_adapter.rs+++ b/datafusion/core/tests/parquet/schema_adapter.rs@@ -482,7 +482,7 @@ fn test_apply_schema_adapter_with_factory() {]));// Create a parquet source- let source = ParquetSource::default();+ let source = ParquetSource::new(schema.clone());// Create a file scan config with source that has a schema adapter factorylet factory = Arc::new(PrefixAdapterFactory {@@ -491,12 +491,9 @@ fn test_apply_schema_adapter_with_factory() {let file_source = source.clone().with_schema_adapter_factory(factory).unwrap();- let config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- schema.clone(),- file_source,- )- .build();+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .build();// Apply schema adapter to a new sourcelet result_source = source.apply_schema_adapter(&config).unwrap();@@ -532,18 +529,15 @@ fn test_apply_schema_adapter_without_factory() {]));// Create a parquet source- let source = ParquetSource::default();+ let source = ParquetSource::new(schema.clone());// Convert to Arc<dyn FileSource>let file_source: Arc<dyn FileSource> = Arc::new(source.clone());// Create a file scan config without a schema adapter factory- let config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- schema.clone(),- file_source,- )- .build();+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .build();// Apply schema adapter function - should pass through the source unchangedlet result_source = source.apply_schema_adapter(&config).unwrap();diff --git a/datafusion/core/tests/parquet/schema_coercion.rs b/datafusion/core/tests/parquet/schema_coercion.rsindex 9be391a910..51e5242cba 100644--- a/datafusion/core/tests/parquet/schema_coercion.rs+++ b/datafusion/core/tests/parquet/schema_coercion.rs@@ -62,14 +62,10 @@ async fn multi_parquet_coercion() {Field::new("c2", DataType::Int32, true),Field::new("c3", DataType::Float64, true),]));- let source = Arc::new(ParquetSource::default());- let conf = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- file_schema,- source,- )- .with_file_group(file_group)- .build();+ let source = Arc::new(ParquetSource::new(file_schema.clone()));+ let conf = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file_group(file_group)+ .build();let parquet_exec = DataSourceExec::from_data_source(conf);@@ -122,8 +118,7 @@ async fn multi_parquet_coercion_projection() {]));let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),- file_schema,- Arc::new(ParquetSource::default()),+ Arc::new(ParquetSource::new(file_schema)),).with_file_group(file_group).with_projection_indices(Some(vec![1, 0, 2]))diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex 5b7d9ac8fb..f0f610dfba 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -37,6 +37,7 @@ use datafusion::datasource::physical_plan::{CsvSource, ParquetSource};use datafusion::datasource::source::DataSourceExec;use datafusion::datasource::MemTable;use datafusion::prelude::{SessionConfig, SessionContext};+use datafusion_common::config::CsvOptions;use datafusion_common::error::Result;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::ScalarValue;@@ -229,8 +230,7 @@ fn parquet_exec_multiple_sorted() -> Arc<DataSourceExec> {let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- schema(),- Arc::new(ParquetSource::default()),+ Arc::new(ParquetSource::new(schema())),).with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new("x".to_string(), 100)]),@@ -247,14 +247,19 @@ fn csv_exec() -> Arc<DataSourceExec> {}fn csv_exec_with_sort(output_ordering: Vec<LexOrdering>) -> Arc<DataSourceExec> {- let config = FileScanConfigBuilder::new(- ObjectStoreUrl::parse("test:///").unwrap(),- schema(),- Arc::new(CsvSource::new(false, b',', b'"')),- )- .with_file(PartitionedFile::new("x".to_string(), 100))- .with_output_ordering(output_ordering)- .build();+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(), {+ let options = CsvOptions {+ has_header: Some(false),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ Arc::new(CsvSource::new(schema()).with_csv_options(options))+ })+ .with_file(PartitionedFile::new("x".to_string(), 100))+ .with_output_ordering(output_ordering)+ .build();DataSourceExec::from_data_source(config)}@@ -265,17 +270,22 @@ fn csv_exec_multiple() -> Arc<DataSourceExec> {// Created a sorted parquet exec with multiple filesfn csv_exec_multiple_sorted(output_ordering: Vec<LexOrdering>) -> Arc<DataSourceExec> {- let config = FileScanConfigBuilder::new(- ObjectStoreUrl::parse("test:///").unwrap(),- schema(),- Arc::new(CsvSource::new(false, b',', b'"')),- )- .with_file_groups(vec![- FileGroup::new(vec![PartitionedFile::new("x".to_string(), 100)]),- FileGroup::new(vec![PartitionedFile::new("y".to_string(), 100)]),- ])- .with_output_ordering(output_ordering)- .build();+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(), {+ let options = CsvOptions {+ has_header: Some(false),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ Arc::new(CsvSource::new(schema()).with_csv_options(options))+ })+ .with_file_groups(vec![+ FileGroup::new(vec![PartitionedFile::new("x".to_string(), 100)]),+ FileGroup::new(vec![PartitionedFile::new("y".to_string(), 100)]),+ ])+ .with_output_ordering(output_ordering)+ .build();DataSourceExec::from_data_source(config)}@@ -2597,11 +2607,15 @@ fn parallelization_compressed_csv() -> Result<()> {for compression_type in compression_types {let plan = aggregate_exec_with_alias(DataSourceExec::from_data_source(- FileScanConfigBuilder::new(- ObjectStoreUrl::parse("test:///").unwrap(),- schema(),- Arc::new(CsvSource::new(false, b',', b'"')),- )+ FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(), {+ let options = CsvOptions {+ has_header: Some(false),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ Arc::new(CsvSource::new(schema()).with_csv_options(options))+ }).with_file(PartitionedFile::new("x".to_string(), 100)).with_file_compression_type(compression_type).build(),diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex e3a0eb7e1a..c0cfa46733 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -31,7 +31,7 @@ use crate::physical_optimizer::test_utils::{use arrow::compute::SortOptions;use arrow::datatypes::{DataType, SchemaRef};-use datafusion_common::config::ConfigOptions;+use datafusion_common::config::{ConfigOptions, CsvOptions};use datafusion_common::tree_node::{TreeNode, TransformedResult};use datafusion_common::{Result, TableReference};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;@@ -72,10 +72,15 @@ fn csv_exec_sorted(schema: &SchemaRef,sort_exprs: impl IntoIterator<Item = PhysicalSortExpr>,) -> Arc<dyn ExecutionPlan> {+ let options = CsvOptions {+ has_header: Some(false),+ delimiter: 0,+ quote: 0,+ ..Default::default()+ };let mut builder = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- schema.clone(),- Arc::new(CsvSource::new(false, 0, 0)),+ Arc::new(CsvSource::new(schema.clone()).with_csv_options(options)),).with_file(PartitionedFile::new("x".to_string(), 100));if let Some(ordering) = LexOrdering::new(sort_exprs) {diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rsindex de61149508..31909415a2 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs@@ -859,20 +859,17 @@ async fn test_topk_filter_passes_through_coalesce_partitions() {];// Create a source that supports all batches- let source = Arc::new(TestSource::new(true, batches));-- let base_config = FileScanConfigBuilder::new(- ObjectStoreUrl::parse("test://").unwrap(),- Arc::clone(&schema()),- source,- )- .with_file_groups(vec![- // Partition 0- FileGroup::new(vec![PartitionedFile::new("test1.parquet", 123)]),- // Partition 1- FileGroup::new(vec![PartitionedFile::new("test2.parquet", 123)]),- ])- .build();+ let source = Arc::new(TestSource::new(schema(), true, batches));++ let base_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::parse("test://").unwrap(), source)+ .with_file_groups(vec![+ // Partition 0+ FileGroup::new(vec![PartitionedFile::new("test1.parquet", 123)]),+ // Partition 1+ FileGroup::new(vec![PartitionedFile::new("test2.parquet", 123)]),+ ])+ .build();let scan = DataSourceExec::from_data_source(base_config);diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rsindex 7d8a9c7c21..2bd70221f4 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs@@ -24,7 +24,6 @@ use datafusion_datasource::{file_scan_config::FileScanConfigBuilder, file_stream::FileOpenFuture,file_stream::FileOpener, schema_adapter::DefaultSchemaAdapterFactory,schema_adapter::SchemaAdapterFactory, source::DataSourceExec, PartitionedFile,- TableSchema,};use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_optimizer::PhysicalOptimizerRule;@@ -53,7 +52,7 @@ use std::{pub struct TestOpener {batches: Vec<RecordBatch>,batch_size: Option<usize>,- schema: Option<SchemaRef>,+ schema: SchemaRef,projection: Option<Vec<usize>>,predicate: Option<Arc<dyn PhysicalExpr>>,}@@ -71,23 +70,23 @@ impl FileOpener for TestOpener {}batches = new_batches.into_iter().collect();}- if let Some(schema) = &self.schema {- let factory = DefaultSchemaAdapterFactory::from_schema(Arc::clone(schema));- let (mapper, projection) = factory.map_schema(&batches[0].schema()).unwrap();- let mut new_batches = Vec::new();- for batch in batches {- let batch = if let Some(predicate) = &self.predicate {- batch_filter(&batch, predicate)?- } else {- batch- };- let batch = batch.project(&projection).unwrap();- let batch = mapper.map_batch(batch).unwrap();- new_batches.push(batch);- }- batches = new_batches;+ let factory = DefaultSchemaAdapterFactory::from_schema(Arc::clone(&self.schema));+ let (mapper, projection) = factory.map_schema(&batches[0].schema()).unwrap();+ let mut new_batches = Vec::new();+ for batch in batches {+ let batch = if let Some(predicate) = &self.predicate {+ batch_filter(&batch, predicate)?+ } else {+ batch+ };++ let batch = batch.project(&projection).unwrap();+ let batch = mapper.map_batch(batch).unwrap();+ new_batches.push(batch);}+ batches = new_batches;+if let Some(projection) = &self.projection {batches = batches.into_iter()@@ -102,26 +101,35 @@ impl FileOpener for TestOpener {}/// A placeholder data source that accepts filter pushdown-#[derive(Clone, Default)]+#[derive(Clone)]pub struct TestSource {support: bool,predicate: Option<Arc<dyn PhysicalExpr>>,statistics: Option<Statistics>,batch_size: Option<usize>,batches: Vec<RecordBatch>,- schema: Option<SchemaRef>,+ schema: SchemaRef,metrics: ExecutionPlanMetricsSet,projection: Option<Vec<usize>>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,+ table_schema: datafusion_datasource::TableSchema,}impl TestSource {- pub fn new(support: bool, batches: Vec<RecordBatch>) -> Self {+ pub fn new(schema: SchemaRef, support: bool, batches: Vec<RecordBatch>) -> Self {+ let table_schema =+ datafusion_datasource::TableSchema::new(Arc::clone(&schema), vec![]);Self {+ schema,support,metrics: ExecutionPlanMetricsSet::new(),batches,- ..Default::default()+ predicate: None,+ statistics: None,+ batch_size: None,+ projection: None,+ schema_adapter_factory: None,+ table_schema,}}}@@ -136,7 +144,7 @@ impl FileSource for TestSource {Arc::new(TestOpener {batches: self.batches.clone(),batch_size: self.batch_size,- schema: self.schema.clone(),+ schema: Arc::clone(&self.schema),projection: self.projection.clone(),predicate: self.predicate.clone(),})@@ -157,17 +165,6 @@ impl FileSource for TestSource {})}- fn with_schema(&self, schema: TableSchema) -> Arc<dyn FileSource> {- assert!(- schema.table_partition_cols().is_empty(),- "TestSource does not support partition columns"- );- Arc::new(TestSource {- schema: Some(schema.file_schema().clone()),- ..self.clone()- })- }-fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource> {Arc::new(TestSource {projection: config.projection_exprs.as_ref().map(|p| p.column_indices()),@@ -260,6 +257,10 @@ impl FileSource for TestSource {fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {self.schema_adapter_factory.clone()}++ fn table_schema(&self) -> &datafusion_datasource::TableSchema {+ &self.table_schema+ }}#[derive(Debug, Clone)]@@ -289,14 +290,15 @@ impl TestScanBuilder {}pub fn build(self) -> Arc<dyn ExecutionPlan> {- let source = Arc::new(TestSource::new(self.support, self.batches));- let base_config = FileScanConfigBuilder::new(- ObjectStoreUrl::parse("test://").unwrap(),+ let source = Arc::new(TestSource::new(Arc::clone(&self.schema),- source,- )- .with_file(PartitionedFile::new("test.parquet", 123))- .build();+ self.support,+ self.batches,+ ));+ let base_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::parse("test://").unwrap(), source)+ .with_file(PartitionedFile::new("test.parquet", 123))+ .build();DataSourceExec::from_data_source(base_config)}}diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rsindex 8631613c39..9d39a80fb9 100644--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs@@ -24,9 +24,10 @@ use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::physical_plan::CsvSource;use datafusion::datasource::source::DataSourceExec;-use datafusion_common::config::ConfigOptions;+use datafusion_common::config::{ConfigOptions, CsvOptions};use datafusion_common::{JoinSide, JoinType, NullEquality, Result, ScalarValue};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;+use datafusion_datasource::TableSchema;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::{@@ -384,14 +385,19 @@ fn create_simple_csv_exec() -> Arc<dyn ExecutionPlan> {Field::new("d", DataType::Int32, true),Field::new("e", DataType::Int32, true),]));- let config = FileScanConfigBuilder::new(- ObjectStoreUrl::parse("test:///").unwrap(),- schema,- Arc::new(CsvSource::new(false, 0, 0)),- )- .with_file(PartitionedFile::new("x".to_string(), 100))- .with_projection_indices(Some(vec![0, 1, 2, 3, 4]))- .build();+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(), {+ let options = CsvOptions {+ has_header: Some(false),+ delimiter: 0,+ quote: 0,+ ..Default::default()+ };+ Arc::new(CsvSource::new(schema.clone()).with_csv_options(options))+ })+ .with_file(PartitionedFile::new("x".to_string(), 100))+ .with_projection_indices(Some(vec![0, 1, 2, 3, 4]))+ .build();DataSourceExec::from_data_source(config)}@@ -403,14 +409,19 @@ fn create_projecting_csv_exec() -> Arc<dyn ExecutionPlan> {Field::new("c", DataType::Int32, true),Field::new("d", DataType::Int32, true),]));- let config = FileScanConfigBuilder::new(- ObjectStoreUrl::parse("test:///").unwrap(),- schema,- Arc::new(CsvSource::new(false, 0, 0)),- )- .with_file(PartitionedFile::new("x".to_string(), 100))- .with_projection_indices(Some(vec![3, 2, 1]))- .build();+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(), {+ let options = CsvOptions {+ has_header: Some(false),+ delimiter: 0,+ quote: 0,+ ..Default::default()+ };+ Arc::new(CsvSource::new(schema.clone()).with_csv_options(options))+ })+ .with_file(PartitionedFile::new("x".to_string(), 100))+ .with_projection_indices(Some(vec![3, 2, 1]))+ .build();DataSourceExec::from_data_source(config)}@@ -1589,13 +1600,21 @@ fn partitioned_data_source() -> Arc<DataSourceExec> {Field::new("string_col", DataType::Utf8, true),]));+ let options = CsvOptions {+ has_header: Some(false),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let table_schema = TableSchema::new(+ Arc::clone(&file_schema),+ vec![Arc::new(Field::new("partition_col", DataType::Utf8, true))],+ );let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- file_schema.clone(),- Arc::new(CsvSource::default()),+ Arc::new(CsvSource::new(table_schema).with_csv_options(options)),).with_file(PartitionedFile::new("x".to_string(), 100))- .with_table_partition_cols(vec![Field::new("partition_col", DataType::Utf8, true)]).with_projection_indices(Some(vec![0, 1, 2])).build();diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex 8ca33f3d4a..60fec22436 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -73,8 +73,7 @@ use datafusion_physical_plan::{pub fn parquet_exec(schema: SchemaRef) -> Arc<DataSourceExec> {let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- schema,- Arc::new(ParquetSource::default()),+ Arc::new(ParquetSource::new(schema)),).with_file(PartitionedFile::new("x".to_string(), 100)).build();@@ -89,8 +88,7 @@ pub(crate) fn parquet_exec_with_sort() -> Arc<DataSourceExec> {let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- schema,- Arc::new(ParquetSource::default()),+ Arc::new(ParquetSource::new(schema)),).with_file(PartitionedFile::new("x".to_string(), 100)).with_output_ordering(output_ordering)@@ -127,8 +125,7 @@ pub(crate) fn parquet_exec_with_stats(file_size: u64) -> Arc<DataSourceExec> {let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- schema(),- Arc::new(ParquetSource::new(Default::default())),+ Arc::new(ParquetSource::new(schema())),).with_file(PartitionedFile::new("x".to_string(), file_size)).with_statistics(statistics)diff --git a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rsindex c3c92a9028..0b093485c1 100644--- a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs+++ b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs@@ -27,12 +27,14 @@ use datafusion::datasource::physical_plan::{};use datafusion::physical_plan::ExecutionPlan;use datafusion::prelude::SessionContext;+use datafusion_common::config::CsvOptions;use datafusion_common::ColumnStatistics;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::schema_adapter::{SchemaAdapter, SchemaAdapterFactory, SchemaMapper,};use datafusion_datasource::source::DataSourceExec;+use datafusion_datasource::TableSchema;use datafusion_execution::object_store::ObjectStoreUrl;use object_store::{memory::InMemory, path::Path, ObjectStore};use parquet::arrow::ArrowWriter;@@ -182,17 +184,17 @@ async fn test_parquet_integration_with_schema_adapter() -> Result<()> {let ctx = SessionContext::new();ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));- // Create a ParquetSource with the adapter factory- let file_source = ParquetSource::default()- .with_schema_adapter_factory(Arc::new(UppercaseAdapterFactory {}))?;-// Create a table schema with uppercase column nameslet table_schema = Arc::new(Schema::new(vec![Field::new("ID", DataType::Int32, false),Field::new("NAME", DataType::Utf8, true),]));- let config = FileScanConfigBuilder::new(store_url, table_schema.clone(), file_source)+ // Create a ParquetSource with the adapter factory+ let file_source = ParquetSource::new(table_schema.clone())+ .with_schema_adapter_factory(Arc::new(UppercaseAdapterFactory {}))?;++ let config = FileScanConfigBuilder::new(store_url, file_source).with_file(PartitionedFile::new(path, file_size)).build();@@ -245,10 +247,10 @@ async fn test_parquet_integration_with_schema_adapter_and_expression_rewriter(ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));// Create a ParquetSource with the adapter factory- let file_source = ParquetSource::default()+ let file_source = ParquetSource::new(batch.schema()).with_schema_adapter_factory(Arc::new(UppercaseAdapterFactory {}))?;- let config = FileScanConfigBuilder::new(store_url, batch.schema(), file_source)+ let config = FileScanConfigBuilder::new(store_url, file_source).with_file(PartitionedFile::new(path, file_size)).build();@@ -284,7 +286,10 @@ async fn test_multi_source_schema_adapter_reuse() -> Result<()> {// Test ArrowSource{- let source = ArrowSource::default();+ let schema =+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));+ let table_schema = TableSchema::new(schema, vec![]);+ let source = ArrowSource::new(table_schema);let source_with_adapter = source.clone().with_schema_adapter_factory(factory.clone())@@ -304,7 +309,9 @@ async fn test_multi_source_schema_adapter_reuse() -> Result<()> {// Test ParquetSource#[cfg(feature = "parquet")]{- let source = ParquetSource::default();+ let schema =+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));+ let source = ParquetSource::new(schema);let source_with_adapter = source.clone().with_schema_adapter_factory(factory.clone())@@ -323,7 +330,15 @@ async fn test_multi_source_schema_adapter_reuse() -> Result<()> {// Test CsvSource{- let source = CsvSource::default();+ let schema =+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));+ let options = CsvOptions {+ has_header: Some(true),+ delimiter: b',',+ quote: b'"',+ ..Default::default()+ };+ let source = CsvSource::new(schema).with_csv_options(options);let source_with_adapter = source.clone().with_schema_adapter_factory(factory.clone())@@ -342,7 +357,10 @@ async fn test_multi_source_schema_adapter_reuse() -> Result<()> {// Test JsonSource{- let source = JsonSource::default();+ let schema =+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));+ let table_schema = TableSchema::new(schema, vec![]);+ let source = JsonSource::new(table_schema);let source_with_adapter = source.clone().with_schema_adapter_factory(factory.clone())diff --git a/datafusion/datasource-arrow/src/file_format.rs b/datafusion/datasource-arrow/src/file_format.rsindex 3b85640804..dc1f5cf72d 100644--- a/datafusion/datasource-arrow/src/file_format.rs+++ b/datafusion/datasource-arrow/src/file_format.rs@@ -45,6 +45,7 @@ use datafusion_datasource::sink::{DataSink, DataSinkExec};use datafusion_datasource::write::{get_writer_schema, ObjectWriterBuilder, SharedBuffer,};+use datafusion_datasource::TableSchema;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::dml::InsertOp;use datafusion_physical_expr_common::sort_expr::LexRequirement;@@ -178,7 +179,11 @@ impl FileFormat for ArrowFormat {_state: &dyn Session,conf: FileScanConfig,) -> Result<Arc<dyn ExecutionPlan>> {- let source = Arc::new(ArrowSource::default());+ let table_schema = TableSchema::new(+ Arc::clone(conf.file_schema()),+ conf.table_partition_cols().clone(),+ );+ let source = Arc::new(ArrowSource::new(table_schema));let config = FileScanConfigBuilder::from(conf).with_source(source).build();@@ -202,8 +207,8 @@ impl FileFormat for ArrowFormat {Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)}- fn file_source(&self) -> Arc<dyn FileSource> {- Arc::new(ArrowSource::default())+ fn file_source(&self, table_schema: TableSchema) -> Arc<dyn FileSource> {+ Arc::new(ArrowSource::new(table_schema))}}diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rsindex f254b7e3ff..b3253d43f4 100644--- a/datafusion/datasource-arrow/src/source.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -20,7 +20,6 @@ use std::sync::Arc;use datafusion_datasource::as_file_source;use datafusion_datasource::schema_adapter::SchemaAdapterFactory;-use datafusion_datasource::TableSchema;use arrow::buffer::Buffer;use arrow_ipc::reader::FileDecoder;@@ -39,13 +38,26 @@ use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore};/// Arrow configuration struct that is given to DataSourceExec/// Does not hold anything special, since [`FileScanConfig`] is sufficient for arrow-#[derive(Clone, Default)]+#[derive(Clone)]pub struct ArrowSource {+ table_schema: datafusion_datasource::TableSchema,metrics: ExecutionPlanMetricsSet,projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}+impl ArrowSource {+ /// Initialize an ArrowSource with the provided schema+ pub fn new(table_schema: impl Into<datafusion_datasource::TableSchema>) -> Self {+ Self {+ table_schema: table_schema.into(),+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,+ }+ }+}+impl From<ArrowSource> for Arc<dyn FileSource> {fn from(source: ArrowSource) -> Self {as_file_source(source)@@ -69,13 +81,14 @@ impl FileSource for ArrowSource {self}- fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {- Arc::new(Self { ..self.clone() })+ fn table_schema(&self) -> &datafusion_datasource::TableSchema {+ &self.table_schema}- fn with_schema(&self, _schema: TableSchema) -> Arc<dyn FileSource> {+ fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {Arc::new(Self { ..self.clone() })}+fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {let mut conf = self.clone();conf.projected_statistics = Some(statistics);diff --git a/datafusion/datasource-avro/src/file_format.rs b/datafusion/datasource-avro/src/file_format.rsindex 60c361b42e..50aecf97b2 100644--- a/datafusion/datasource-avro/src/file_format.rs+++ b/datafusion/datasource-avro/src/file_format.rs@@ -154,13 +154,17 @@ impl FileFormat for AvroFormat {_state: &dyn Session,conf: FileScanConfig,) -> Result<Arc<dyn ExecutionPlan>> {+ let file_schema = Arc::clone(conf.file_schema());let config = FileScanConfigBuilder::from(conf)- .with_source(self.file_source())+ .with_source(Arc::new(AvroSource::new(file_schema))).build();Ok(DataSourceExec::from_data_source(config))}- fn file_source(&self) -> Arc<dyn FileSource> {- Arc::new(AvroSource::new())+ fn file_source(+ &self,+ table_schema: datafusion_datasource::TableSchema,+ ) -> Arc<dyn FileSource> {+ Arc::new(AvroSource::new(table_schema))}}diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rsindex 1ff73d2c3c..9859e11e25 100644--- a/datafusion/datasource-avro/src/source.rs+++ b/datafusion/datasource-avro/src/source.rs@@ -22,7 +22,6 @@ use std::sync::Arc;use crate::avro_to_arrow::Reader as AvroReader;-use arrow::datatypes::SchemaRef;use datafusion_common::error::Result;use datafusion_common::Statistics;use datafusion_datasource::file::FileSource;@@ -36,9 +35,9 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;use object_store::ObjectStore;/// AvroSource holds the extra configuration that is necessary for opening avro files-#[derive(Clone, Default)]+#[derive(Clone)]pub struct AvroSource {- schema: Option<SchemaRef>,+ table_schema: TableSchema,batch_size: Option<usize>,projection: Option<Vec<String>>,metrics: ExecutionPlanMetricsSet,@@ -47,15 +46,22 @@ pub struct AvroSource {}impl AvroSource {- /// Initialize an AvroSource with default values- pub fn new() -> Self {- Self::default()+ /// Initialize an AvroSource with the provided schema+ pub fn new(table_schema: impl Into<TableSchema>) -> Self {+ Self {+ table_schema: table_schema.into(),+ batch_size: None,+ projection: None,+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,+ }}fn open<R: std::io::Read>(&self, reader: R) -> Result<AvroReader<'static, R>> {AvroReader::try_new(reader,- Arc::clone(self.schema.as_ref().expect("Schema must set before open")),+ Arc::clone(self.table_schema.file_schema()),self.batch_size.expect("Batch size must set before open"),self.projection.clone(),)@@ -79,16 +85,13 @@ impl FileSource for AvroSource {self}- fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.batch_size = Some(batch_size);- Arc::new(conf)+ fn table_schema(&self) -> &TableSchema {+ &self.table_schema}- fn with_schema(&self, schema: TableSchema) -> Arc<dyn FileSource> {+ fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {let mut conf = self.clone();- // TableSchema may have partition columns, but AvroSource does not use partition columns or values atm- conf.schema = Some(Arc::clone(schema.file_schema()));+ conf.batch_size = Some(batch_size);Arc::new(conf)}diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rsindex 1c39893b23..6b27687a56 100644--- a/datafusion/datasource-csv/src/file_format.rs+++ b/datafusion/datasource-csv/src/file_format.rs@@ -48,6 +48,7 @@ use datafusion_datasource::sink::{DataSink, DataSinkExec};use datafusion_datasource::write::demux::DemuxedStreamReceiver;use datafusion_datasource::write::orchestration::spawn_writer_tasks_and_join;use datafusion_datasource::write::BatchSerializer;+use datafusion_datasource::TableSchema;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::dml::InsertOp;use datafusion_physical_expr_common::sort_expr::LexRequirement;@@ -434,20 +435,23 @@ impl FileFormat for CsvFormat {.newlines_in_values.unwrap_or_else(|| state.config_options().catalog.newlines_in_values);- let conf_builder = FileScanConfigBuilder::from(conf)- .with_file_compression_type(self.options.compression.into())- .with_newlines_in_values(newlines_in_values);+ let mut csv_options = self.options.clone();+ csv_options.has_header = Some(has_header);- let truncated_rows = self.options.truncated_rows.unwrap_or(false);- let source = Arc::new(- CsvSource::new(has_header, self.options.delimiter, self.options.quote)- .with_escape(self.options.escape)- .with_terminator(self.options.terminator)- .with_comment(self.options.comment)- .with_truncate_rows(truncated_rows),- );+ // Get the existing CsvSource and update its options+ // We need to preserve the table_schema from the original source (which includes partition columns)+ let csv_source = conf+ .file_source+ .as_any()+ .downcast_ref::<CsvSource>()+ .expect("file_source should be a CsvSource");+ let source = Arc::new(csv_source.clone().with_csv_options(csv_options));- let config = conf_builder.with_source(source).build();+ let config = FileScanConfigBuilder::from(conf)+ .with_file_compression_type(self.options.compression.into())+ .with_newlines_in_values(newlines_in_values)+ .with_source(source)+ .build();Ok(DataSourceExec::from_data_source(config))}@@ -489,8 +493,12 @@ impl FileFormat for CsvFormat {Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)}- fn file_source(&self) -> Arc<dyn FileSource> {- Arc::new(CsvSource::default())+ fn file_source(&self, table_schema: TableSchema) -> Arc<dyn FileSource> {+ let mut csv_options = self.options.clone();+ if csv_options.has_header.is_none() {+ csv_options.has_header = Some(true);+ }+ Arc::new(CsvSource::new(table_schema).with_csv_options(csv_options))}}diff --git a/datafusion/datasource-csv/src/mod.rs b/datafusion/datasource-csv/src/mod.rsindex 90538d0808..78a916912c 100644--- a/datafusion/datasource-csv/src/mod.rs+++ b/datafusion/datasource-csv/src/mod.rs@@ -24,7 +24,6 @@ pub mod source;use std::sync::Arc;-use arrow::datatypes::SchemaRef;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::{file::FileSource, file_scan_config::FileScanConfig};@@ -33,11 +32,10 @@ pub use file_format::*;/// Returns a [`FileScanConfig`] for given `file_groups`pub fn partitioned_csv_config(- schema: SchemaRef,file_groups: Vec<FileGroup>,file_source: Arc<dyn FileSource>,) -> FileScanConfig {- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), schema, file_source)+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source).with_file_groups(file_groups).build()}diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex 0b18571e58..94c6b3810a 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -33,7 +33,7 @@ use datafusion_datasource::{};use arrow::csv;-use arrow::datatypes::SchemaRef;+use datafusion_common::config::CsvOptions;use datafusion_common::{DataFusionError, Result, Statistics};use datafusion_common_runtime::JoinSet;use datafusion_datasource::file::FileSource;@@ -61,111 +61,118 @@ use tokio::io::AsyncWriteExt;/// # use datafusion_datasource_csv::source::CsvSource;/// # use datafusion_execution::object_store::ObjectStoreUrl;/// # use datafusion_datasource::source::DataSourceExec;+/// # use datafusion_common::config::CsvOptions;////// # let object_store_url = ObjectStoreUrl::local_filesystem();/// # let file_schema = Arc::new(Schema::empty());///-/// let source = Arc::new(CsvSource::new(-/// true,-/// b',',-/// b'"',-/// )-/// .with_terminator(Some(b'#')-/// ));+/// let options = CsvOptions {+/// has_header: Some(true),+/// delimiter: b',',+/// quote: b'"',+/// ..Default::default()+/// };+/// let source = Arc::new(CsvSource::new(file_schema.clone())+/// .with_csv_options(options)+/// .with_terminator(Some(b'#'))+/// );/// // Create a DataSourceExec for reading the first 100MB of `file1.csv`-/// let config = FileScanConfigBuilder::new(object_store_url, file_schema, source)+/// let config = FileScanConfigBuilder::new(object_store_url, source)/// .with_file(PartitionedFile::new("file1.csv", 100*1024*1024))/// .with_newlines_in_values(true) // The file contains newlines in values;/// .build();/// let exec = (DataSourceExec::from_data_source(config));/// ```-#[derive(Debug, Clone, Default)]+#[derive(Debug, Clone)]pub struct CsvSource {+ options: CsvOptions,batch_size: Option<usize>,- file_schema: Option<SchemaRef>,+ table_schema: TableSchema,file_projection: Option<Vec<usize>>,- pub(crate) has_header: bool,- delimiter: u8,- quote: u8,- terminator: Option<u8>,- escape: Option<u8>,- comment: Option<u8>,metrics: ExecutionPlanMetricsSet,projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,- truncate_rows: bool,}impl CsvSource {/// Returns a [`CsvSource`]- pub fn new(has_header: bool, delimiter: u8, quote: u8) -> Self {+ pub fn new(table_schema: impl Into<TableSchema>) -> Self {Self {- has_header,- delimiter,- quote,- ..Self::default()+ options: CsvOptions::default(),+ table_schema: table_schema.into(),+ batch_size: None,+ file_projection: None,+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,}}+ /// Sets the CSV options+ pub fn with_csv_options(mut self, options: CsvOptions) -> Self {+ self.options = options;+ self+ }+/// true if the first line of each file is a headerpub fn has_header(&self) -> bool {- self.has_header+ self.options.has_header.unwrap_or(true)}// true if rows length support truncatepub fn truncate_rows(&self) -> bool {- self.truncate_rows+ self.options.truncated_rows.unwrap_or(false)}/// A column delimiterpub fn delimiter(&self) -> u8 {- self.delimiter+ self.options.delimiter}/// The quote characterpub fn quote(&self) -> u8 {- self.quote+ self.options.quote}/// The line terminatorpub fn terminator(&self) -> Option<u8> {- self.terminator+ self.options.terminator}/// Lines beginning with this byte are ignored.pub fn comment(&self) -> Option<u8> {- self.comment+ self.options.comment}/// The escape characterpub fn escape(&self) -> Option<u8> {- self.escape+ self.options.escape}/// Initialize a CsvSource with escapepub fn with_escape(&self, escape: Option<u8>) -> Self {let mut conf = self.clone();- conf.escape = escape;+ conf.options.escape = escape;conf}/// Initialize a CsvSource with terminatorpub fn with_terminator(&self, terminator: Option<u8>) -> Self {let mut conf = self.clone();- conf.terminator = terminator;+ conf.options.terminator = terminator;conf}/// Initialize a CsvSource with commentpub fn with_comment(&self, comment: Option<u8>) -> Self {let mut conf = self.clone();- conf.comment = comment;+ conf.options.comment = comment;conf}/// Whether to support truncate rows when read csv filepub fn with_truncate_rows(&self, truncate_rows: bool) -> Self {let mut conf = self.clone();- conf.truncate_rows = truncate_rows;+ conf.options.truncated_rows = Some(truncate_rows);conf}}@@ -176,29 +183,26 @@ impl CsvSource {}fn builder(&self) -> csv::ReaderBuilder {- let mut builder = csv::ReaderBuilder::new(Arc::clone(- self.file_schema- .as_ref()- .expect("Schema must be set before initializing builder"),- ))- .with_delimiter(self.delimiter)- .with_batch_size(- self.batch_size- .expect("Batch size must be set before initializing builder"),- )- .with_header(self.has_header)- .with_quote(self.quote)- .with_truncated_rows(self.truncate_rows);- if let Some(terminator) = self.terminator {+ let mut builder =+ csv::ReaderBuilder::new(Arc::clone(self.table_schema.file_schema()))+ .with_delimiter(self.delimiter())+ .with_batch_size(+ self.batch_size+ .expect("Batch size must be set before initializing builder"),+ )+ .with_header(self.has_header())+ .with_quote(self.quote())+ .with_truncated_rows(self.truncate_rows());+ if let Some(terminator) = self.terminator() {builder = builder.with_terminator(terminator);}if let Some(proj) = &self.file_projection {builder = builder.with_projection(proj.clone());}- if let Some(escape) = self.escape {+ if let Some(escape) = self.escape() {builder = builder.with_escape(escape)}- if let Some(comment) = self.comment {+ if let Some(comment) = self.comment() {builder = builder.with_comment(comment);}@@ -252,15 +256,13 @@ impl FileSource for CsvSource {self}- fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.batch_size = Some(batch_size);- Arc::new(conf)+ fn table_schema(&self) -> &TableSchema {+ &self.table_schema}- fn with_schema(&self, schema: TableSchema) -> Arc<dyn FileSource> {+ fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {let mut conf = self.clone();- conf.file_schema = Some(Arc::clone(schema.file_schema()));+ conf.batch_size = Some(batch_size);Arc::new(conf)}@@ -291,7 +293,7 @@ impl FileSource for CsvSource {fn fmt_extra(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {match t {DisplayFormatType::Default | DisplayFormatType::Verbose => {- write!(f, ", has_header={}", self.has_header)+ write!(f, ", has_header={}", self.has_header())}DisplayFormatType::TreeRender => Ok(()),}@@ -340,18 +342,16 @@ impl FileOpener for CsvOpener {// `self.config.has_header` controls whether to skip reading the 1st line header// If the .csv file is read in parallel and this `CsvOpener` is only reading some middle// partition, then don't skip first line- let mut csv_has_header = self.config.has_header;+ let mut csv_has_header = self.config.has_header();if let Some(FileRange { start, .. }) = partitioned_file.range {if start != 0 {csv_has_header = false;}}- let config = CsvSource {- has_header: csv_has_header,- truncate_rows: self.config.truncate_rows,- ..(*self.config).clone()- };+ let mut config = (*self.config).clone();+ config.options.has_header = Some(csv_has_header);+ config.options.truncated_rows = Some(config.truncate_rows());let file_compression_type = self.file_compression_type.to_owned();@@ -363,7 +363,7 @@ impl FileOpener for CsvOpener {}let store = Arc::clone(&self.object_store);- let terminator = self.config.terminator;+ let terminator = self.config.terminator();Ok(Box::pin(async move {// Current partition contains bytes [start_byte, end_byte) (might contain incomplete lines at boundaries)diff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rsindex 51f4bd7e96..afb12e5262 100644--- a/datafusion/datasource-json/src/file_format.rs+++ b/datafusion/datasource-json/src/file_format.rs@@ -50,6 +50,7 @@ use datafusion_datasource::sink::{DataSink, DataSinkExec};use datafusion_datasource::write::demux::DemuxedStreamReceiver;use datafusion_datasource::write::orchestration::spawn_writer_tasks_and_join;use datafusion_datasource::write::BatchSerializer;+use datafusion_datasource::TableSchema;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::dml::InsertOp;use datafusion_physical_expr_common::sort_expr::LexRequirement;@@ -253,7 +254,11 @@ impl FileFormat for JsonFormat {_state: &dyn Session,conf: FileScanConfig,) -> Result<Arc<dyn ExecutionPlan>> {- let source = Arc::new(JsonSource::new());+ let table_schema = TableSchema::new(+ Arc::clone(conf.file_schema()),+ conf.table_partition_cols().clone(),+ );+ let source = Arc::new(JsonSource::new(table_schema));let conf = FileScanConfigBuilder::from(conf).with_file_compression_type(FileCompressionType::from(self.options.compression,@@ -281,8 +286,8 @@ impl FileFormat for JsonFormat {Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)}- fn file_source(&self) -> Arc<dyn FileSource> {- Arc::new(JsonSource::default())+ fn file_source(&self, table_schema: TableSchema) -> Arc<dyn FileSource> {+ Arc::new(JsonSource::new(table_schema))}}diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex 52ed0def03..44b71ce680 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -32,7 +32,6 @@ use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_datasource::{as_file_source, calculate_range, ListingTableUrl, PartitionedFile, RangeCalculation,- TableSchema,};use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};@@ -75,8 +74,9 @@ impl JsonOpener {}/// JsonSource holds the extra configuration that is necessary for [`JsonOpener`]-#[derive(Clone, Default)]+#[derive(Clone)]pub struct JsonSource {+ table_schema: datafusion_datasource::TableSchema,batch_size: Option<usize>,metrics: ExecutionPlanMetricsSet,projected_statistics: Option<Statistics>,@@ -84,9 +84,15 @@ pub struct JsonSource {}impl JsonSource {- /// Initialize a JsonSource with default values- pub fn new() -> Self {- Self::default()+ /// Initialize a JsonSource with the provided schema+ pub fn new(table_schema: impl Into<datafusion_datasource::TableSchema>) -> Self {+ Self {+ table_schema: table_schema.into(),+ batch_size: None,+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,+ }}}@@ -117,15 +123,16 @@ impl FileSource for JsonSource {self}+ fn table_schema(&self) -> &datafusion_datasource::TableSchema {+ &self.table_schema+ }+fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {let mut conf = self.clone();conf.batch_size = Some(batch_size);Arc::new(conf)}- fn with_schema(&self, _schema: TableSchema) -> Arc<dyn FileSource> {- Arc::new(Self { ..self.clone() })- }fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {let mut conf = self.clone();conf.projected_statistics = Some(statistics);diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex f27bda387f..1e86d41927 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -32,6 +32,7 @@ use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};use datafusion_datasource::write::{get_writer_schema, ObjectWriterBuilder, SharedBuffer,};+use datafusion_datasource::TableSchema;use datafusion_datasource::file_format::{FileFormat, FileFormatFactory};use datafusion_datasource::write::demux::DemuxedStreamReceiver;@@ -459,7 +460,12 @@ impl FileFormat for ParquetFormat {metadata_size_hint = Some(metadata);}- let mut source = ParquetSource::new(self.options.clone());+ let table_schema = TableSchema::new(+ Arc::clone(conf.file_schema()),+ conf.table_partition_cols().clone(),+ );+ let mut source = ParquetSource::new(table_schema)+ .with_table_parquet_options(self.options.clone());// Use the CachedParquetFileReaderFactorylet metadata_cache = state.runtime_env().cache_manager.get_file_metadata_cache();@@ -501,8 +507,11 @@ impl FileFormat for ParquetFormat {Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)}- fn file_source(&self) -> Arc<dyn FileSource> {- Arc::new(ParquetSource::default())+ fn file_source(&self, table_schema: TableSchema) -> Arc<dyn FileSource> {+ Arc::new(+ ParquetSource::new(table_schema)+ .with_table_parquet_options(self.options.clone()),+ )}}diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 2815b82f1d..3c905d950a 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -270,7 +270,7 @@ impl FileOpener for ParquetOpener {let partition_values = partition_fields.iter().cloned()- .zip(partitioned_file.partition_values)+ .zip(partitioned_file.partition_values.clone()).collect_vec();let expr = expr_adapter_factory.create(diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 450ccc5d06..27640f37ce 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -104,11 +104,11 @@ use parquet::encryption::decrypt::FileDecryptionProperties;/// # let object_store_url = ObjectStoreUrl::local_filesystem();/// # let predicate = lit(true);/// let source = Arc::new(-/// ParquetSource::default()-/// .with_predicate(predicate)+/// ParquetSource::new(Arc::clone(&file_schema))+/// .with_predicate(predicate)/// );/// // Create a DataSourceExec for reading `file1.parquet` with a file size of 100MB-/// let config = FileScanConfigBuilder::new(object_store_url, file_schema, source)+/// let config = FileScanConfigBuilder::new(object_store_url, source)/// .with_file(PartitionedFile::new("file1.parquet", 100*1024*1024)).build();/// let exec = DataSourceExec::from_data_source(config);/// ```@@ -231,7 +231,7 @@ use parquet::encryption::decrypt::FileDecryptionProperties;/// let partitioned_file = PartitionedFile::new("my_file.parquet", 1234)/// .with_extensions(Arc::new(access_plan));/// // create a FileScanConfig to scan this file-/// let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), schema(), Arc::new(ParquetSource::default()))+/// let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), Arc::new(ParquetSource::new(schema())))/// .with_file(partitioned_file).build();/// // this parquet DataSourceExec will not even try to read row groups 2 and 4. Additional/// // pruning based on predicates may also happen@@ -266,7 +266,7 @@ use parquet::encryption::decrypt::FileDecryptionProperties;/// [`RecordBatch`]: arrow::record_batch::RecordBatch/// [`SchemaAdapter`]: datafusion_datasource::schema_adapter::SchemaAdapter/// [`ParquetMetadata`]: parquet::file::metadata::ParquetMetaData-#[derive(Clone, Default, Debug)]+#[derive(Clone, Debug)]pub struct ParquetSource {/// Options for reading Parquet filespub(crate) table_parquet_options: TableParquetOptions,@@ -275,7 +275,7 @@ pub struct ParquetSource {/// The schema of the file./// In particular, this is the schema of the table without partition columns,/// *not* the physical schema of the file.- pub(crate) table_schema: Option<TableSchema>,+ pub(crate) table_schema: TableSchema,/// Optional predicate for row filtering during parquet scanpub(crate) predicate: Option<Arc<dyn PhysicalExpr>>,/// Optional user defined parquet file reader factory@@ -293,15 +293,35 @@ pub struct ParquetSource {impl ParquetSource {/// Create a new ParquetSource to read the data specified in the file scan- /// configuration with the provided `TableParquetOptions`.- /// if default values are going to be used, use `ParguetConfig::default()` instead- pub fn new(table_parquet_options: TableParquetOptions) -> Self {+ /// configuration with the provided schema.+ ///+ /// Uses default `TableParquetOptions`.+ /// To set custom options, use [ParquetSource::with_table_parquet_options`].+ pub fn new(table_schema: impl Into<TableSchema>) -> Self {Self {- table_parquet_options,- ..Self::default()+ table_schema: table_schema.into(),+ table_parquet_options: TableParquetOptions::default(),+ metrics: ExecutionPlanMetricsSet::new(),+ predicate: None,+ parquet_file_reader_factory: None,+ schema_adapter_factory: None,+ batch_size: None,+ metadata_size_hint: None,+ projected_statistics: None,+ #[cfg(feature = "parquet_encryption")]+ encryption_factory: None,}}+ /// Set the `TableParquetOptions` for this ParquetSource.+ pub fn with_table_parquet_options(+ mut self,+ table_parquet_options: TableParquetOptions,+ ) -> Self {+ self.table_parquet_options = table_parquet_options;+ self+ }+/// Set the metadata size hint////// This value determines how many bytes at the end of the file the default@@ -590,6 +610,10 @@ impl FileSource for ParquetSource {self}+ fn table_schema(&self) -> &TableSchema {+ &self.table_schema+ }+fn filter(&self) -> Option<Arc<dyn PhysicalExpr>> {self.predicate.clone()}@@ -600,13 +624,6 @@ impl FileSource for ParquetSource {Arc::new(conf)}- fn with_schema(&self, schema: TableSchema) -> Arc<dyn FileSource> {- Arc::new(Self {- table_schema: Some(schema),- ..self.clone()- })- }-fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {let mut conf = self.clone();conf.projected_statistics = Some(statistics);@@ -660,14 +677,11 @@ impl FileSource for ParquetSource {// the actual predicates are built in reference to the physical schema of// each file, which we do not have at this point and hence cannot use.// Instead we use the logical schema of the file (the table schema without partition columns).- if let (Some(file_schema), Some(predicate)) = (- &self.table_schema.as_ref().map(|ts| ts.file_schema()),- &self.predicate,- ) {+ if let Some(predicate) = &self.predicate {let predicate_creation_errors = Count::new();if let (Some(pruning_predicate), _) = build_pruning_predicates(Some(predicate),- file_schema,+ self.table_schema.table_schema(),&predicate_creation_errors,) {let mut guarantees = pruning_predicate@@ -700,16 +714,7 @@ impl FileSource for ParquetSource {filters: Vec<Arc<dyn PhysicalExpr>>,config: &ConfigOptions,) -> datafusion_common::Result<FilterPushdownPropagation<Arc<dyn FileSource>>> {- let Some(table_schema) = self- .table_schema- .as_ref()- .map(|ts| ts.table_schema())- .cloned()- else {- return Ok(FilterPushdownPropagation::with_parent_pushdown_result(- vec![PushedDown::No; filters.len()],- ));- };+ let table_schema = self.table_schema.table_schema();// Determine if based on configs we should push filters down.// If either the table / scan itself or the config has pushdown enabled,// we will push down the filters.@@ -725,7 +730,7 @@ impl FileSource for ParquetSource {let filters: Vec<PushedDownPredicate> = filters.into_iter().map(|filter| {- if can_expr_be_pushed_down_with_schemas(&filter, &table_schema) {+ if can_expr_be_pushed_down_with_schemas(&filter, table_schema) {PushedDownPredicate::supported(filter)} else {PushedDownPredicate::unsupported(filter)@@ -790,6 +795,7 @@ impl FileSource for ParquetSource {#[cfg(test)]mod tests {use super::*;+ use arrow::datatypes::Schema;use datafusion_physical_expr::expressions::lit;#[test]@@ -797,7 +803,8 @@ mod tests {fn test_parquet_source_predicate_same_as_filter() {let predicate = lit(true);- let parquet_source = ParquetSource::default().with_predicate(predicate);+ let parquet_source =+ ParquetSource::new(Arc::new(Schema::empty())).with_predicate(predicate);// same value. but filter() call Arc::clone internallyassert_eq!(parquet_source.predicate(), parquet_source.filter().as_ref());}diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex d6ade3b8b2..9245f60e23 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -26,7 +26,6 @@ use crate::file_groups::FileGroupPartitioner;use crate::file_scan_config::FileScanConfig;use crate::file_stream::FileOpener;use crate::schema_adapter::SchemaAdapterFactory;-use crate::TableSchema;use datafusion_common::config::ConfigOptions;use datafusion_common::{not_impl_err, Result, Statistics};use datafusion_physical_expr::{LexOrdering, PhysicalExpr};@@ -61,10 +60,12 @@ pub trait FileSource: Send + Sync {) -> Arc<dyn FileOpener>;/// Anyfn as_any(&self) -> &dyn Any;+ /// Returns the table schema for this file source.+ ///+ /// This always returns the unprojected schema (the full schema of the data).+ fn table_schema(&self) -> &crate::table_schema::TableSchema;/// Initialize new type with batch size configurationfn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>;- /// Initialize new instance with a new schema- fn with_schema(&self, schema: TableSchema) -> Arc<dyn FileSource>;/// Initialize new instance with projection informationfn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource>;/// Initialize new instance with projected statisticsdiff --git a/datafusion/datasource/src/file_format.rs b/datafusion/datasource/src/file_format.rsindex 23f68636c1..bb4ffded80 100644--- a/datafusion/datasource/src/file_format.rs+++ b/datafusion/datasource/src/file_format.rs@@ -111,7 +111,10 @@ pub trait FileFormat: Send + Sync + fmt::Debug {}/// Return the related FileSource such as `CsvSource`, `JsonSource`, etc.- fn file_source(&self) -> Arc<dyn FileSource>;+ ///+ /// # Arguments+ /// * `table_schema` - The table schema to use for the FileSource (includes partition columns)+ fn file_source(&self, table_schema: crate::TableSchema) -> Arc<dyn FileSource>;}/// Factory for creating [`FileFormat`] instances based on session and command level optionsdiff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex 5847a8cf5e..82052ee4c3 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -24,7 +24,7 @@ use crate::schema_adapter::SchemaAdapterFactory;use crate::{display::FileGroupsDisplay, file::FileSource,file_compression_type::FileCompressionType, file_stream::FileStream,- source::DataSource, statistics::MinMaxStatistics, PartitionedFile, TableSchema,+ source::DataSource, statistics::MinMaxStatistics, PartitionedFile,};use arrow::datatypes::FieldRef;use arrow::{@@ -33,7 +33,7 @@ use arrow::{RecordBatchOptions,},buffer::Buffer,- datatypes::{ArrowNativeType, DataType, Field, Schema, SchemaRef, UInt16Type},+ datatypes::{ArrowNativeType, DataType, Schema, SchemaRef, UInt16Type},};use datafusion_common::config::ConfigOptions;use datafusion_common::{@@ -103,29 +103,30 @@ use log::{debug, warn};/// # // Note: crate mock ParquetSource, as ParquetSource is not in the datasource crate/// #[derive(Clone)]/// # struct ParquetSource {+/// # table_schema: TableSchema,/// # projected_statistics: Option<Statistics>,/// # schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>/// # };/// # impl FileSource for ParquetSource {/// # fn create_file_opener(&self, _: Arc<dyn ObjectStore>, _: &FileScanConfig, _: usize) -> Arc<dyn FileOpener> { unimplemented!() }/// # fn as_any(&self) -> &dyn Any { self }+/// # fn table_schema(&self) -> &TableSchema { &self.table_schema }/// # fn with_batch_size(&self, _: usize) -> Arc<dyn FileSource> { unimplemented!() }-/// # fn with_schema(&self, _: TableSchema) -> Arc<dyn FileSource> { Arc::new(self.clone()) as Arc<dyn FileSource> }/// # fn with_projection(&self, _: &FileScanConfig) -> Arc<dyn FileSource> { unimplemented!() }-/// # fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> { Arc::new(Self {projected_statistics: Some(statistics), schema_adapter_factory: self.schema_adapter_factory.clone()} ) }+/// # fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> { Arc::new(Self {table_schema: self.table_schema.clone(), projected_statistics: Some(statistics), schema_adapter_factory: self.schema_adapter_factory.clone()} ) }/// # fn metrics(&self) -> &ExecutionPlanMetricsSet { unimplemented!() }/// # fn statistics(&self) -> Result<Statistics> { Ok(self.projected_statistics.clone().expect("projected_statistics should be set")) }/// # fn file_type(&self) -> &str { "parquet" }-/// # fn with_schema_adapter_factory(&self, factory: Arc<dyn SchemaAdapterFactory>) -> Result<Arc<dyn FileSource>> { Ok(Arc::new(Self {projected_statistics: self.projected_statistics.clone(), schema_adapter_factory: Some(factory)} )) }+/// # fn with_schema_adapter_factory(&self, factory: Arc<dyn SchemaAdapterFactory>) -> Result<Arc<dyn FileSource>> { Ok(Arc::new(Self {table_schema: self.table_schema.clone(), projected_statistics: self.projected_statistics.clone(), schema_adapter_factory: Some(factory)} )) }/// # fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> { self.schema_adapter_factory.clone() }/// # }/// # impl ParquetSource {-/// # fn new() -> Self { Self {projected_statistics: None, schema_adapter_factory: None} }+/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into(), projected_statistics: None, schema_adapter_factory: None} }/// # }/// // create FileScan config for reading parquet files from file:///// let object_store_url = ObjectStoreUrl::local_filesystem();-/// let file_source = Arc::new(ParquetSource::new());-/// let config = FileScanConfigBuilder::new(object_store_url, file_schema, file_source)+/// let file_source = Arc::new(ParquetSource::new(file_schema.clone()));+/// let config = FileScanConfigBuilder::new(object_store_url, file_source)/// .with_limit(Some(1000)) // read only the first 1000 records/// .with_projection_indices(Some(vec![2, 3])) // project columns 2 and 3/// // Read /tmp/file1.parquet with known size of 1234 bytes in a single group@@ -156,16 +157,6 @@ pub struct FileScanConfig {/// [`RuntimeEnv::register_object_store`]: datafusion_execution::runtime_env::RuntimeEnv::register_object_store/// [`RuntimeEnv::object_store`]: datafusion_execution::runtime_env::RuntimeEnv::object_storepub object_store_url: ObjectStoreUrl,- /// Schema information including the file schema, table partition columns,- /// and the combined table schema.- ///- /// The table schema (file schema + partition columns) is the schema exposed- /// upstream of [`FileScanConfig`] (e.g. in [`DataSourceExec`]).- ///- /// See [`TableSchema`] for more information.- ///- /// [`DataSourceExec`]: crate::source::DataSourceExec- pub table_schema: TableSchema,/// List of files to be processed, grouped into partitions////// Each file must have a schema of `file_schema` or a subset. If@@ -214,6 +205,7 @@ pub struct FileScanConfig {/// # use datafusion_datasource::file_compression_type::FileCompressionType;/// # use datafusion_datasource::file_groups::FileGroup;/// # use datafusion_datasource::PartitionedFile;+/// # use datafusion_datasource::table_schema::TableSchema;/// # use datafusion_execution::object_store::ObjectStoreUrl;/// # use datafusion_common::Statistics;/// # use datafusion_datasource::file::FileSource;@@ -221,25 +213,28 @@ pub struct FileScanConfig {/// # fn main() {/// # fn with_source(file_source: Arc<dyn FileSource>) {/// // Create a schema for our Parquet files-/// let schema = Arc::new(Schema::new(vec![+/// let file_schema = Arc::new(Schema::new(vec![/// Field::new("id", DataType::Int32, false),/// Field::new("value", DataType::Utf8, false),/// ]));///+/// // Create partition columns+/// let partition_cols = vec![+/// Arc::new(Field::new("date", DataType::Utf8, false)),+/// ];+///+/// // Create table schema with file schema and partition columns+/// let table_schema = TableSchema::new(file_schema, partition_cols);+////// // Create a builder for scanning Parquet files from a local filesystem/// let config = FileScanConfigBuilder::new(/// ObjectStoreUrl::local_filesystem(),-/// schema,/// file_source,/// )/// // Set a limit of 1000 rows/// .with_limit(Some(1000))/// // Project only the first column/// .with_projection_indices(Some(vec![0]))-/// // Add partition columns-/// .with_table_partition_cols(vec![-/// Field::new("date", DataType::Utf8, false),-/// ])/// // Add a file group with two files/// .with_file_group(FileGroup::new(vec![/// PartitionedFile::new("data/date=2024-01-01/file1.parquet", 1024),@@ -255,16 +250,6 @@ pub struct FileScanConfig {#[derive(Clone)]pub struct FileScanConfigBuilder {object_store_url: ObjectStoreUrl,- /// Schema information including the file schema, table partition columns,- /// and the combined table schema.- ///- /// This schema is used to read the files, but the file schema is **not** necessarily- /// the schema of the physical files. Rather this is the schema that the- /// physical file schema will be mapped onto, and the schema that the- /// [`DataSourceExec`] will return.- ///- /// [`DataSourceExec`]: crate::source::DataSourceExec- table_schema: TableSchema,file_source: Arc<dyn FileSource>,limit: Option<usize>,projection_indices: Option<Vec<usize>>,@@ -283,16 +268,14 @@ impl FileScanConfigBuilder {////// # Parameters:/// * `object_store_url`: See [`FileScanConfig::object_store_url`]- /// * `file_schema`: See [`FileScanConfig::file_schema`]- /// * `file_source`: See [`FileScanConfig::file_source`]+ /// * `file_source`: See [`FileScanConfig::file_source`]. The file source must have+ /// a schema set via its constructor.pub fn new(object_store_url: ObjectStoreUrl,- file_schema: SchemaRef,file_source: Arc<dyn FileSource>,) -> Self {Self {object_store_url,- table_schema: TableSchema::from_file_schema(file_schema),file_source,file_groups: vec![],statistics: None,@@ -324,7 +307,7 @@ impl FileScanConfigBuilder {}pub fn table_schema(&self) -> &SchemaRef {- self.table_schema.table_schema()+ self.file_source.table_schema().table_schema()}/// Set the columns on which to project the data. Indexes that are higher than the@@ -345,18 +328,6 @@ impl FileScanConfigBuilder {self}- /// Set the partitioning columns- pub fn with_table_partition_cols(mut self, table_partition_cols: Vec<Field>) -> Self {- let table_partition_cols: Vec<FieldRef> = table_partition_cols- .into_iter()- .map(|f| Arc::new(f) as FieldRef)- .collect();- self.table_schema = self- .table_schema- .with_table_partition_cols(table_partition_cols);- self- }-/// Set the table constraintspub fn with_constraints(mut self, constraints: Constraints) -> Self {self.constraints = Some(constraints);@@ -451,7 +422,6 @@ impl FileScanConfigBuilder {pub fn build(self) -> FileScanConfig {let Self {object_store_url,- table_schema,file_source,limit,projection_indices,@@ -466,12 +436,11 @@ impl FileScanConfigBuilder {} = self;let constraints = constraints.unwrap_or_default();- let statistics = statistics- .unwrap_or_else(|| Statistics::new_unknown(table_schema.file_schema()));+ let statistics = statistics.unwrap_or_else(|| {+ Statistics::new_unknown(file_source.table_schema().file_schema())+ });- let file_source = file_source- .with_statistics(statistics.clone())- .with_schema(table_schema.clone());+ let file_source = file_source.with_statistics(statistics.clone());let file_compression_type =file_compression_type.unwrap_or(FileCompressionType::UNCOMPRESSED);let new_lines_in_values = new_lines_in_values.unwrap_or(false);@@ -479,12 +448,14 @@ impl FileScanConfigBuilder {// Convert projection indices to ProjectionExprs using the final table schema// (which now includes partition columns if they were added)let projection_exprs = projection_indices.map(|indices| {- ProjectionExprs::from_indices(&indices, table_schema.table_schema())+ ProjectionExprs::from_indices(+ &indices,+ file_source.table_schema().table_schema(),+ )});FileScanConfig {object_store_url,- table_schema,file_source,limit,projection_exprs,@@ -503,7 +474,6 @@ impl From<FileScanConfig> for FileScanConfigBuilder {fn from(config: FileScanConfig) -> Self {Self {object_store_url: config.object_store_url,- table_schema: config.table_schema,file_source: Arc::<dyn FileSource>::clone(&config.file_source),file_groups: config.file_groups,statistics: config.file_source.statistics().ok(),@@ -748,12 +718,12 @@ impl DataSource for FileScanConfig {impl FileScanConfig {/// Get the file schema (schema of the files without partition columns)pub fn file_schema(&self) -> &SchemaRef {- self.table_schema.file_schema()+ self.file_source.table_schema().file_schema()}/// Get the table partition columnspub fn table_partition_cols(&self) -> &Vec<FieldRef> {- self.table_schema.table_partition_cols()+ self.file_source.table_schema().table_partition_cols()}fn projection_indices(&self) -> Vec<usize> {@@ -1509,12 +1479,14 @@ pub fn wrap_partition_value_in_dict(val: ScalarValue) -> ScalarValue {mod tests {use super::*;use crate::test_util::col;+ use crate::TableSchema;use crate::{generate_test_files, test_util::MockSource, tests::aggr_test_schema,verify_sort_integrity,};use arrow::array::{Int32Array, RecordBatch};+ use arrow::datatypes::Field;use datafusion_common::stats::Precision;use datafusion_common::{assert_batches_eq, internal_err};use datafusion_expr::{Operator, SortExpr};@@ -2178,14 +2150,16 @@ mod tests {statistics: Statistics,table_partition_cols: Vec<Field>,) -> FileScanConfig {+ let table_schema = TableSchema::new(+ file_schema,+ table_partition_cols.into_iter().map(Arc::new).collect(),+ );FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- file_schema,- Arc::new(MockSource::default()),+ Arc::new(MockSource::new(table_schema.clone())),).with_projection_indices(projection).with_statistics(statistics)- .with_table_partition_cols(table_partition_cols).build()}@@ -2224,12 +2198,22 @@ mod tests {fn test_file_scan_config_builder() {let file_schema = aggr_test_schema();let object_store_url = ObjectStoreUrl::parse("test:///").unwrap();- let file_source: Arc<dyn FileSource> = Arc::new(MockSource::default());++ let table_schema = TableSchema::new(+ Arc::clone(&file_schema),+ vec![Arc::new(Field::new(+ "date",+ wrap_partition_type_in_dict(DataType::Utf8),+ false,+ ))],+ );++ let file_source: Arc<dyn FileSource> =+ Arc::new(MockSource::new(table_schema.clone()));// Create a builder with required parameterslet builder = FileScanConfigBuilder::new(object_store_url.clone(),- Arc::clone(&file_schema),Arc::clone(&file_source),);@@ -2237,11 +2221,6 @@ mod tests {let config = builder.with_limit(Some(1000)).with_projection_indices(Some(vec![0, 1]))- .with_table_partition_cols(vec![Field::new(- "date",- wrap_partition_type_in_dict(DataType::Utf8),- false,- )]).with_statistics(Statistics::new_unknown(&file_schema)).with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new("test.parquet".to_string(),@@ -2283,17 +2262,20 @@ mod tests {fn equivalence_properties_after_schema_change() {let file_schema = aggr_test_schema();let object_store_url = ObjectStoreUrl::parse("test:///").unwrap();++ let table_schema = TableSchema::new(Arc::clone(&file_schema), vec![]);+// Create a file source with a filter- let file_source: Arc<dyn FileSource> =- Arc::new(MockSource::default().with_filter(Arc::new(BinaryExpr::new(+ let file_source: Arc<dyn FileSource> = Arc::new(+ MockSource::new(table_schema.clone()).with_filter(Arc::new(BinaryExpr::new(col("c2", &file_schema).unwrap(),Operator::Eq,Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),- ))));+ ))),+ );let config = FileScanConfigBuilder::new(object_store_url.clone(),- Arc::clone(&file_schema),Arc::clone(&file_source),).with_projection_indices(Some(vec![0, 1, 2]))@@ -2331,12 +2313,15 @@ mod tests {fn test_file_scan_config_builder_defaults() {let file_schema = aggr_test_schema();let object_store_url = ObjectStoreUrl::parse("test:///").unwrap();- let file_source: Arc<dyn FileSource> = Arc::new(MockSource::default());++ let table_schema = TableSchema::new(Arc::clone(&file_schema), vec![]);++ let file_source: Arc<dyn FileSource> =+ Arc::new(MockSource::new(table_schema.clone()));// Create a builder with only required parameters and build without any additional configurationslet config = FileScanConfigBuilder::new(object_store_url.clone(),- Arc::clone(&file_schema),Arc::clone(&file_source),).build();@@ -2389,7 +2374,6 @@ mod tests {fn test_file_scan_config_builder_new_from() {let schema = aggr_test_schema();let object_store_url = ObjectStoreUrl::parse("test:///").unwrap();- let file_source: Arc<dyn FileSource> = Arc::new(MockSource::default());let partition_cols = vec![Field::new("date",wrap_partition_type_in_dict(DataType::Utf8),@@ -2397,15 +2381,21 @@ mod tests {)];let file = PartitionedFile::new("test_file.parquet", 100);+ let table_schema = TableSchema::new(+ Arc::clone(&schema),+ partition_cols.iter().map(|f| Arc::new(f.clone())).collect(),+ );++ let file_source: Arc<dyn FileSource> =+ Arc::new(MockSource::new(table_schema.clone()));+// Create a config with non-default valueslet original_config = FileScanConfigBuilder::new(object_store_url.clone(),- Arc::clone(&schema),Arc::clone(&file_source),).with_projection_indices(Some(vec![0, 2])).with_limit(Some(10))- .with_table_partition_cols(partition_cols.clone()).with_file(file.clone()).with_constraints(Constraints::default()).with_newlines_in_values(true)@@ -2640,11 +2630,12 @@ mod tests {let file_group = FileGroup::new(vec![PartitionedFile::new("test.parquet", 1024)]).with_statistics(Arc::new(file_group_stats));+ let table_schema = TableSchema::new(Arc::clone(&schema), vec![]);+// Create a FileScanConfig with projection: only keep columns 0 and 2let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- Arc::clone(&schema),- Arc::new(MockSource::default()),+ Arc::new(MockSource::new(table_schema.clone())),).with_projection_indices(Some(vec![0, 2])) // Only project columns 0 and 2.with_file_groups(vec![file_group])diff --git a/datafusion/datasource/src/file_stream.rs b/datafusion/datasource/src/file_stream.rsindex a4a43ca9ae..0568b4cc4e 100644--- a/datafusion/datasource/src/file_stream.rs+++ b/datafusion/datasource/src/file_stream.rs@@ -639,10 +639,10 @@ mod tests {let on_error = self.on_error;+ let table_schema = crate::table_schema::TableSchema::new(file_schema, vec![]);let config = FileScanConfigBuilder::new(ObjectStoreUrl::parse("test:///").unwrap(),- file_schema,- Arc::new(MockSource::default()),+ Arc::new(MockSource::new(table_schema)),).with_file_group(file_group).with_limit(self.limit)diff --git a/datafusion/datasource/src/table_schema.rs b/datafusion/datasource/src/table_schema.rsindex 8002df4a99..ff0e788018 100644--- a/datafusion/datasource/src/table_schema.rs+++ b/datafusion/datasource/src/table_schema.rs@@ -170,3 +170,9 @@ impl TableSchema {&self.table_schema}}++impl From<SchemaRef> for TableSchema {+ fn from(schema: SchemaRef) -> Self {+ Self::from_file_schema(schema)+ }+}diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rsindex feb704af99..78ba593f22 100644--- a/datafusion/datasource/src/test_util.rs+++ b/datafusion/datasource/src/test_util.rs@@ -22,7 +22,6 @@ use crate::{use std::sync::Arc;-use crate::TableSchema;use arrow::datatypes::Schema;use datafusion_common::{Result, Statistics};use datafusion_physical_expr::{expressions::Column, PhysicalExpr};@@ -30,15 +29,41 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;use object_store::ObjectStore;/// Minimal [`crate::file::FileSource`] implementation for use in tests.-#[derive(Clone, Default)]+#[derive(Clone)]pub(crate) struct MockSource {metrics: ExecutionPlanMetricsSet,projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,filter: Option<Arc<dyn PhysicalExpr>>,+ table_schema: crate::table_schema::TableSchema,+}++impl Default for MockSource {+ fn default() -> Self {+ Self {+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,+ filter: None,+ table_schema: crate::table_schema::TableSchema::new(+ Arc::new(Schema::empty()),+ vec![],+ ),+ }+ }}impl MockSource {+ pub fn new(table_schema: impl Into<crate::table_schema::TableSchema>) -> Self {+ Self {+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,+ filter: None,+ table_schema: table_schema.into(),+ }+ }+pub fn with_filter(mut self, filter: Arc<dyn PhysicalExpr>) -> Self {self.filter = Some(filter);self@@ -67,10 +92,6 @@ impl FileSource for MockSource {Arc::new(Self { ..self.clone() })}- fn with_schema(&self, _schema: TableSchema) -> Arc<dyn FileSource> {- Arc::new(Self { ..self.clone() })- }-fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {Arc::new(Self { ..self.clone() })}@@ -110,6 +131,10 @@ impl FileSource for MockSource {fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {self.schema_adapter_factory.clone()}++ fn table_schema(&self) -> &crate::table_schema::TableSchema {+ &self.table_schema+ }}/// Create a column expressiondiff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 349ed79ddb..f1a9abe6ea 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -34,7 +34,7 @@ use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};use datafusion_datasource::file_sink_config::FileSinkConfig;-use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile};+use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile, TableSchema};use datafusion_datasource_csv::file_format::CsvSink;use datafusion_datasource_json::file_format::JsonSink;#[cfg(feature = "parquet")]@@ -481,6 +481,37 @@ pub fn parse_protobuf_file_scan_schema(Ok(Arc::new(convert_required!(proto.schema)?))}+/// Parses a TableSchema from protobuf, extracting the file schema and partition columns+pub fn parse_table_schema_from_proto(+ proto: &protobuf::FileScanExecConf,+) -> Result<TableSchema> {+ let schema: Arc<Schema> = parse_protobuf_file_scan_schema(proto)?;++ // Reacquire the partition column types from the schema before removing them below.+ let table_partition_cols = proto+ .table_partition_cols+ .iter()+ .map(|col| Ok(Arc::new(schema.field_with_name(col)?.clone())))+ .collect::<Result<Vec<_>>>()?;++ // Remove partition columns from the schema after recreating table_partition_cols+ // because the partition columns are not in the file. They are present to allow+ // the partition column types to be reconstructed after serde.+ let file_schema = Arc::new(+ Schema::new(+ schema+ .fields()+ .iter()+ .filter(|field| !table_partition_cols.contains(field))+ .cloned()+ .collect::<Vec<_>>(),+ )+ .with_metadata(schema.metadata.clone()),+ );++ Ok(TableSchema::new(file_schema, table_partition_cols))+}+pub fn parse_protobuf_file_scan_config(proto: &protobuf::FileScanExecConf,ctx: &TaskContext,@@ -508,28 +539,6 @@ pub fn parse_protobuf_file_scan_config(true => ObjectStoreUrl::local_filesystem(),};- // Reacquire the partition column types from the schema before removing them below.- let table_partition_cols = proto- .table_partition_cols- .iter()- .map(|col| Ok(schema.field_with_name(col)?.clone()))- .collect::<Result<Vec<_>>>()?;-- // Remove partition columns from the schema after recreating table_partition_cols- // because the partition columns are not in the file. They are present to allow- // the partition column types to be reconstructed after serde.- let file_schema = Arc::new(- Schema::new(- schema- .fields()- .iter()- .filter(|field| !table_partition_cols.contains(field))- .cloned()- .collect::<Vec<_>>(),- )- .with_metadata(schema.metadata.clone()),- );-let mut output_ordering = vec![];for node_collection in &proto.output_ordering {let sort_exprs = parse_physical_sort_exprs(@@ -541,13 +550,12 @@ pub fn parse_protobuf_file_scan_config(output_ordering.extend(LexOrdering::new(sort_exprs));}- let config = FileScanConfigBuilder::new(object_store_url, file_schema, file_source)+ let config = FileScanConfigBuilder::new(object_store_url, file_source).with_file_groups(file_groups).with_constraints(constraints).with_statistics(statistics).with_projection_indices(Some(projection)).with_limit(proto.limit.as_ref().map(|sl| sl.limit as usize))- .with_table_partition_cols(table_partition_cols).with_output_ordering(output_ordering).with_batch_size(proto.batch_size.map(|s| s as usize)).build();diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex e5f4a1f7d0..fc7818fe46 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -24,6 +24,7 @@ use crate::common::{byte_to_string, str_to_byte};use crate::physical_plan::from_proto::{parse_physical_expr, parse_physical_sort_expr, parse_physical_sort_exprs,parse_physical_window_expr, parse_protobuf_file_scan_config, parse_record_batches,+ parse_table_schema_from_proto,};use crate::physical_plan::to_proto::{serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr,@@ -42,6 +43,7 @@ use crate::{convert_required, into_required};use arrow::compute::SortOptions;use arrow::datatypes::{IntervalMonthDayNanoType, SchemaRef};use datafusion_catalog::memory::MemorySourceConfig;+use datafusion_common::config::CsvOptions;use datafusion_common::{internal_datafusion_err, internal_err, not_impl_err, DataFusionError, Result,};@@ -612,14 +614,21 @@ impl protobuf::PhysicalPlanNode {None};+ // Parse table schema with partition columns+ let table_schema =+ parse_table_schema_from_proto(scan.base_conf.as_ref().unwrap())?;++ let csv_options = CsvOptions {+ has_header: Some(scan.has_header),+ delimiter: str_to_byte(&scan.delimiter, "delimiter")?,+ quote: str_to_byte(&scan.quote, "quote")?,+ ..Default::default()+ };let source = Arc::new(- CsvSource::new(- scan.has_header,- str_to_byte(&scan.delimiter, "delimiter")?,- 0,- )- .with_escape(escape)- .with_comment(comment),+ CsvSource::new(table_schema)+ .with_csv_options(csv_options)+ .with_escape(escape)+ .with_comment(comment),);let conf = FileScanConfigBuilder::from(parse_protobuf_file_scan_config(@@ -641,11 +650,13 @@ impl protobuf::PhysicalPlanNode {extension_codec: &dyn PhysicalExtensionCodec,) -> Result<Arc<dyn ExecutionPlan>> {+ let base_conf = scan.base_conf.as_ref().unwrap();+ let table_schema = parse_table_schema_from_proto(base_conf)?;let scan_conf = parse_protobuf_file_scan_config(- scan.base_conf.as_ref().unwrap(),+ base_conf,ctx,extension_codec,- Arc::new(JsonSource::new()),+ Arc::new(JsonSource::new(table_schema)),)?;Ok(DataSourceExec::from_data_source(scan_conf))}@@ -695,7 +706,12 @@ impl protobuf::PhysicalPlanNode {if let Some(table_options) = scan.parquet_options.as_ref() {options = table_options.try_into()?;}- let mut source = ParquetSource::new(options);++ // Parse table schema with partition columns+ let table_schema = parse_table_schema_from_proto(base_conf)?;++ let mut source =+ ParquetSource::new(table_schema).with_table_parquet_options(options);if let Some(predicate) = predicate {source = source.with_predicate(predicate);@@ -717,16 +733,17 @@ impl protobuf::PhysicalPlanNode {&self,scan: &protobuf::AvroScanExecNode,ctx: &TaskContext,-extension_codec: &dyn PhysicalExtensionCodec,) -> Result<Arc<dyn ExecutionPlan>> {#[cfg(feature = "avro")]{+ let table_schema =+ parse_table_schema_from_proto(scan.base_conf.as_ref().unwrap())?;let conf = parse_protobuf_file_scan_config(scan.base_conf.as_ref().unwrap(),ctx,extension_codec,- Arc::new(AvroSource::new()),+ Arc::new(AvroSource::new(table_schema)),)?;Ok(DataSourceExec::from_data_source(conf))}diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex c8b2bc02e4..73f39eaa7b 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -33,6 +33,7 @@ use arrow::datatypes::{Fields, TimeUnit};use datafusion::physical_expr::aggregate::AggregateExprBuilder;use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;use datafusion::physical_plan::metrics::MetricType;+use datafusion_datasource::TableSchema;use datafusion_expr::dml::InsertOp;use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;use datafusion_functions_aggregate::array_agg::array_agg_udaf;@@ -883,25 +884,26 @@ fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {let mut options = TableParquetOptions::new();options.global.pushdown_filters = true;- let file_source = Arc::new(ParquetSource::new(options).with_predicate(predicate));+ let file_source = Arc::new(+ ParquetSource::new(Arc::clone(&file_schema))+ .with_table_parquet_options(options)+ .with_predicate(predicate),+ );- let scan_config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- file_schema,- file_source,- )- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(Statistics {- num_rows: Precision::Inexact(100),- total_byte_size: Precision::Inexact(1024),- column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(vec![- Field::new("col", DataType::Utf8, false),- ]))),- })- .build();+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(Statistics {+ num_rows: Precision::Inexact(100),+ total_byte_size: Precision::Inexact(1024),+ column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(+ vec![Field::new("col", DataType::Utf8, false)],+ ))),+ })+ .build();roundtrip_test(DataSourceExec::from_data_source(scan_config))}@@ -914,21 +916,22 @@ async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {vec![wrap_partition_value_in_dict(ScalarValue::Int64(Some(0)))];let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let file_source = Arc::new(ParquetSource::default());- let scan_config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- schema,- file_source,- )- .with_projection_indices(Some(vec![0, 1]))- .with_file_group(FileGroup::new(vec![file_group]))- .with_table_partition_cols(vec![Field::new(- "part".to_string(),- wrap_partition_type_in_dict(DataType::Int16),- false,- )])- .with_newlines_in_values(false)- .build();+ let table_schema = TableSchema::new(+ schema.clone(),+ vec![Arc::new(Field::new(+ "part".to_string(),+ wrap_partition_type_in_dict(DataType::Int16),+ false,+ ))],+ );++ let file_source = Arc::new(ParquetSource::new(table_schema.clone()));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_projection_indices(Some(vec![0, 1]))+ .with_file_group(FileGroup::new(vec![file_group]))+ .with_newlines_in_values(false)+ .build();roundtrip_test(DataSourceExec::from_data_source(scan_config))}@@ -942,26 +945,25 @@ fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {inner: Arc::new(Column::new("col", 1)),});- let file_source =- Arc::new(ParquetSource::default().with_predicate(custom_predicate_expr));+ let file_source = Arc::new(+ ParquetSource::new(Arc::clone(&file_schema))+ .with_predicate(custom_predicate_expr),+ );- let scan_config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- file_schema,- file_source,- )- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(Statistics {- num_rows: Precision::Inexact(100),- total_byte_size: Precision::Inexact(1024),- column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(vec![- Field::new("col", DataType::Utf8, false),- ]))),- })- .build();+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(Statistics {+ num_rows: Precision::Inexact(100),+ total_byte_size: Precision::Inexact(1024),+ column_statistics: Statistics::unknown_column(&Arc::new(Schema::new(+ vec![Field::new("col", DataType::Utf8, false)],+ ))),+ })+ .build();#[derive(Debug, Clone, Eq)]struct CustomPredicateExpr {@@ -1803,19 +1805,17 @@ async fn roundtrip_projection_source() -> Result<()> {let statistics = Statistics::new_unknown(&schema);- let file_source = ParquetSource::default().with_statistics(statistics.clone());- let scan_config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- schema.clone(),- file_source,- )- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.parquet".to_string(),- 1024,- )])])- .with_statistics(statistics)- .with_projection_indices(Some(vec![0, 1, 2]))- .build();+ let file_source =+ ParquetSource::new(Arc::clone(&schema)).with_statistics(statistics.clone());+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_statistics(statistics)+ .with_projection_indices(Some(vec![0, 1, 2]))+ .build();let filter = Arc::new(FilterExec::try_new(diff --git a/datafusion/substrait/src/physical_plan/consumer.rs b/datafusion/substrait/src/physical_plan/consumer.rsindex 45a19cea80..8ce71acecc 100644--- a/datafusion/substrait/src/physical_plan/consumer.rs+++ b/datafusion/substrait/src/physical_plan/consumer.rs@@ -53,7 +53,6 @@ pub async fn from_substrait_rel() -> Result<Arc<dyn ExecutionPlan>> {let mut base_config_builder;- let source = Arc::new(ParquetSource::default());match &rel.rel_type {Some(RelType::Read(read)) => {if read.filter.is_some() || read.best_effort_filter.is_some() {@@ -80,9 +79,10 @@ pub async fn from_substrait_rel(.collect::<Result<Vec<Field>>>(){Ok(fields) => {+ let schema = Arc::new(Schema::new(fields));+ let source = Arc::new(ParquetSource::new(Arc::clone(&schema)));base_config_builder = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),- Arc::new(Schema::new(fields)),source,);}diff --git a/datafusion/substrait/tests/cases/roundtrip_physical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_physical_plan.rsindex 64599465f9..bafaffa828 100644--- a/datafusion/substrait/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_physical_plan.rs@@ -35,24 +35,22 @@ use substrait::proto::extensions;#[tokio::test]async fn parquet_exec() -> Result<()> {- let source = Arc::new(ParquetSource::default());-- let scan_config = FileScanConfigBuilder::new(- ObjectStoreUrl::local_filesystem(),- Arc::new(Schema::empty()),- source,- )- .with_file_groups(vec![- FileGroup::new(vec![PartitionedFile::new(- "file://foo/part-0.parquet".to_string(),- 123,- )]),- FileGroup::new(vec![PartitionedFile::new(- "file://foo/part-1.parquet".to_string(),- 123,- )]),- ])- .build();+ let schema = Arc::new(Schema::empty());+ let source = Arc::new(ParquetSource::new(schema.clone()));++ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file_groups(vec![+ FileGroup::new(vec![PartitionedFile::new(+ "file://foo/part-0.parquet".to_string(),+ 123,+ )]),+ FileGroup::new(vec![PartitionedFile::new(+ "file://foo/part-1.parquet".to_string(),+ 123,+ )]),+ ])+ .build();let parquet_exec: Arc<dyn ExecutionPlan> =DataSourceExec::from_data_source(scan_config);diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex 0b227000f7..f08e2c383a 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -150,7 +150,7 @@ let projection_exprs = config.projection_exprs;The `FileScanConfigBuilder::with_projection()` method has been deprecated in favor of `with_projection_indices()`:```diff-let config = FileScanConfigBuilder::new(url, schema, file_source)+let config = FileScanConfigBuilder::new(url, file_source)- .with_projection(Some(vec![0, 2, 3]))+ .with_projection_indices(Some(vec![0, 2, 3])).build();@@ -190,6 +190,91 @@ TIMEZONE = '+00:00';This change was made to better support using the default timezone in scalar UDF functions such as`now`, `current_date`, `current_time`, and `to_timestamp` among others.+### Refactoring of `FileSource` constructors and `FileScanConfigBuilder` to accept schemas upfront++The way schemas are passed to file sources and scan configurations has been significantly refactored. File sources now require the schema (including partition columns) to be provided at construction time, and `FileScanConfigBuilder` no longer takes a separate schema parameter.++**Who is affected:**++- Users who create `FileScanConfig` or file sources (`ParquetSource`, `CsvSource`, `JsonSource`, `AvroSource`) directly+- Users who implement custom `FileFormat` implementations++**Key changes:**++1. **FileSource constructors now require TableSchema**: All built-in file sources now take the schema in their constructor:++ ```diff+ - let source = ParquetSource::default();+ + let source = ParquetSource::new(table_schema);+ ```++2. **FileScanConfigBuilder no longer takes schema as a parameter**: The schema is now passed via the FileSource:++ ```diff+ - FileScanConfigBuilder::new(url, schema, source)+ + FileScanConfigBuilder::new(url, source)+ ```++3. **Partition columns are now part of TableSchema**: The `with_table_partition_cols()` method has been removed from `FileScanConfigBuilder`. Partition columns are now passed as part of the `TableSchema` to the FileSource constructor:++ ```diff+ + let table_schema = TableSchema::new(+ + file_schema,+ + vec![Arc::new(Field::new("date", DataType::Utf8, false))],+ + );+ + let source = ParquetSource::new(table_schema);+ let config = FileScanConfigBuilder::new(url, source)+ - .with_table_partition_cols(vec![Field::new("date", DataType::Utf8, false)])+ .with_file(partitioned_file)+ .build();+ ```++4. **FileFormat::file_source() now takes TableSchema parameter**: Custom `FileFormat` implementations must be updated:+ ```diff+ impl FileFormat for MyFileFormat {+ - fn file_source(&self) -> Arc<dyn FileSource> {+ + fn file_source(&self, table_schema: TableSchema) -> Arc<dyn FileSource> {+ - Arc::new(MyFileSource::default())+ + Arc::new(MyFileSource::new(table_schema))+ }+ }+ ```++**Migration examples:**++For Parquet files:++```diff+- let source = Arc::new(ParquetSource::default());+- let config = FileScanConfigBuilder::new(url, schema, source)++ let table_schema = TableSchema::new(schema, vec![]);++ let source = Arc::new(ParquetSource::new(table_schema));++ let config = FileScanConfigBuilder::new(url, source)+ .with_file(partitioned_file)+ .build();+```++For CSV files with partition columns:++```diff+- let source = Arc::new(CsvSource::new(true, b',', b'"'));+- let config = FileScanConfigBuilder::new(url, file_schema, source)+- .with_table_partition_cols(vec![Field::new("year", DataType::Int32, false)])++ let options = CsvOptions {++ has_header: Some(true),++ delimiter: b',',++ quote: b'"',++ ..Default::default()++ };++ let table_schema = TableSchema::new(++ file_schema,++ vec![Arc::new(Field::new("year", DataType::Int32, false))],++ );++ let source = Arc::new(CsvSource::new(table_schema).with_csv_options(options));++ let config = FileScanConfigBuilder::new(url, source)+ .build();+```+### Introduction of `TableSchema` and changes to `FileSource::with_schema()` methodA new `TableSchema` struct has been introduced in the `datafusion-datasource` crate to better manage table schemas with partition columns. This struct helps distinguish between:@@ -1137,7 +1222,7 @@ Pattern in DataFusion `47.0.0`:```rust# /* comment to avoid running-let config = FileScanConfigBuilder::new(url, schema, Arc::new(file_source))+let config = FileScanConfigBuilder::new(url, Arc::new(file_source)).with_statistics(stats)....build();