Apache DataFusion SQL Query Engine
refactor: move ListingTable over to the catalog-listing-table crate (#18080)
Tim Saucer3272ebe9989f
diff --git a/Cargo.lock b/Cargo.lockindex 7b09121595..0392c8147a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1913,12 +1913,15 @@ dependencies = ["datafusion-catalog","datafusion-common","datafusion-datasource",+ "datafusion-datasource-parquet","datafusion-execution","datafusion-expr","datafusion-physical-expr",+ "datafusion-physical-expr-adapter","datafusion-physical-expr-common","datafusion-physical-plan","futures",+ "itertools 0.14.0","log","object_store","tokio",diff --git a/datafusion-examples/examples/custom_file_casts.rs b/datafusion-examples/examples/custom_file_casts.rsindex 65ca096820..4d97ecd91d 100644--- a/datafusion-examples/examples/custom_file_casts.rs+++ b/datafusion-examples/examples/custom_file_casts.rs@@ -25,7 +25,7 @@ use datafusion::common::not_impl_err;use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion::common::{Result, ScalarValue};use datafusion::datasource::listing::{- ListingTable, ListingTableConfig, ListingTableUrl,+ ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,};use datafusion::execution::context::SessionContext;use datafusion::execution::object_store::ObjectStoreUrl;diff --git a/datafusion-examples/examples/json_shredding.rs b/datafusion-examples/examples/json_shredding.rsindex c7d0146a00..a2e83bc951 100644--- a/datafusion-examples/examples/json_shredding.rs+++ b/datafusion-examples/examples/json_shredding.rs@@ -27,7 +27,7 @@ use datafusion::common::tree_node::{};use datafusion::common::{assert_contains, exec_datafusion_err, Result};use datafusion::datasource::listing::{- ListingTable, ListingTableConfig, ListingTableUrl,+ ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,};use datafusion::execution::context::SessionContext;use datafusion::execution::object_store::ObjectStoreUrl;diff --git a/datafusion/catalog-listing/Cargo.toml b/datafusion/catalog-listing/Cargo.tomlindex 69f952ae98..4eaeed675a 100644--- a/datafusion/catalog-listing/Cargo.toml+++ b/datafusion/catalog-listing/Cargo.toml@@ -39,14 +39,17 @@ datafusion-datasource = { workspace = true }datafusion-execution = { workspace = true }datafusion-expr = { workspace = true }datafusion-physical-expr = { workspace = true }+datafusion-physical-expr-adapter = { workspace = true }datafusion-physical-expr-common = { workspace = true }datafusion-physical-plan = { workspace = true }futures = { workspace = true }+itertools = { workspace = true }log = { workspace = true }object_store = { workspace = true }tokio = { workspace = true }[dev-dependencies]+datafusion-datasource-parquet = { workspace = true }[lints]workspace = true@@ -54,3 +57,6 @@ workspace = true[lib]name = "datafusion_catalog_listing"path = "src/mod.rs"++[package.metadata.cargo-machete]+ignored = ["datafusion-datasource-parquet"]diff --git a/datafusion/catalog-listing/src/config.rs b/datafusion/catalog-listing/src/config.rsnew file mode 100644index 0000000000..90f44de4fd--- /dev/null+++ b/datafusion/catalog-listing/src/config.rs@@ -0,0 +1,360 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use crate::options::ListingOptions;+use arrow::datatypes::{DataType, Schema, SchemaRef};+use datafusion_catalog::Session;+use datafusion_common::{config_err, internal_err};+use datafusion_datasource::file_compression_type::FileCompressionType;+use datafusion_datasource::schema_adapter::SchemaAdapterFactory;+use datafusion_datasource::ListingTableUrl;+use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;+use std::str::FromStr;+use std::sync::Arc;++/// Indicates the source of the schema for a [`crate::ListingTable`]+// PartialEq required for assert_eq! in tests+#[derive(Debug, Clone, Copy, PartialEq, Default)]+pub enum SchemaSource {+ /// Schema is not yet set (initial state)+ #[default]+ Unset,+ /// Schema was inferred from first table_path+ Inferred,+ /// Schema was specified explicitly via with_schema+ Specified,+}++/// Configuration for creating a [`crate::ListingTable`]+///+/// # Schema Evolution Support+///+/// This configuration supports schema evolution through the optional+/// [`SchemaAdapterFactory`]. You might want to override the default factory when you need:+///+/// - **Type coercion requirements**: When you need custom logic for converting between+/// different Arrow data types (e.g., Int32 ↔ Int64, Utf8 ↔ LargeUtf8)+/// - **Column mapping**: You need to map columns with a legacy name to a new name+/// - **Custom handling of missing columns**: By default they are filled in with nulls, but you may e.g. want to fill them in with `0` or `""`.+///+/// If not specified, a [`datafusion_datasource::schema_adapter::DefaultSchemaAdapterFactory`]+/// will be used, which handles basic schema compatibility cases.+///+#[derive(Debug, Clone, Default)]+pub struct ListingTableConfig {+ /// Paths on the `ObjectStore` for creating [`crate::ListingTable`].+ /// They should share the same schema and object store.+ pub table_paths: Vec<ListingTableUrl>,+ /// Optional `SchemaRef` for the to be created [`crate::ListingTable`].+ ///+ /// See details on [`ListingTableConfig::with_schema`]+ pub file_schema: Option<SchemaRef>,+ /// Optional [`ListingOptions`] for the to be created [`crate::ListingTable`].+ ///+ /// See details on [`ListingTableConfig::with_listing_options`]+ pub options: Option<ListingOptions>,+ /// Tracks the source of the schema information+ pub(crate) schema_source: SchemaSource,+ /// Optional [`SchemaAdapterFactory`] for creating schema adapters+ pub(crate) schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,+ /// Optional [`PhysicalExprAdapterFactory`] for creating physical expression adapters+ pub(crate) expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,+}++impl ListingTableConfig {+ /// Creates new [`ListingTableConfig`] for reading the specified URL+ pub fn new(table_path: ListingTableUrl) -> Self {+ Self {+ table_paths: vec![table_path],+ ..Default::default()+ }+ }++ /// Creates new [`ListingTableConfig`] with multiple table paths.+ ///+ /// See `ListingTableConfigExt::infer_options` for details on what happens with multiple paths+ pub fn new_with_multi_paths(table_paths: Vec<ListingTableUrl>) -> Self {+ Self {+ table_paths,+ ..Default::default()+ }+ }++ /// Returns the source of the schema for this configuration+ pub fn schema_source(&self) -> SchemaSource {+ self.schema_source+ }+ /// Set the `schema` for the overall [`crate::ListingTable`]+ ///+ /// [`crate::ListingTable`] will automatically coerce, when possible, the schema+ /// for individual files to match this schema.+ ///+ /// If a schema is not provided, it is inferred using+ /// [`Self::infer_schema`].+ ///+ /// If the schema is provided, it must contain only the fields in the file+ /// without the table partitioning columns.+ ///+ /// # Example: Specifying Table Schema+ /// ```rust+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::{ListingTableConfig, ListingOptions};+ /// # use datafusion_datasource::ListingTableUrl;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ /// # use arrow::datatypes::{Schema, Field, DataType};+ /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap();+ /// # let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default()));+ /// let schema = Arc::new(Schema::new(vec![+ /// Field::new("id", DataType::Int64, false),+ /// Field::new("name", DataType::Utf8, true),+ /// ]));+ ///+ /// let config = ListingTableConfig::new(table_paths)+ /// .with_listing_options(listing_options) // Set options first+ /// .with_schema(schema); // Then set schema+ /// ```+ pub fn with_schema(self, schema: SchemaRef) -> Self {+ // Note: We preserve existing options state, but downstream code may expect+ // options to be set. Consider calling with_listing_options() or infer_options()+ // before operations that require options to be present.+ debug_assert!(+ self.options.is_some() || cfg!(test),+ "ListingTableConfig::with_schema called without options set. \+ Consider calling with_listing_options() or infer_options() first to avoid panics in downstream code."+ );++ Self {+ file_schema: Some(schema),+ schema_source: SchemaSource::Specified,+ ..self+ }+ }++ /// Add `listing_options` to [`ListingTableConfig`]+ ///+ /// If not provided, format and other options are inferred via+ /// `ListingTableConfigExt::infer_options`.+ ///+ /// # Example: Configuring Parquet Files with Custom Options+ /// ```rust+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::{ListingTableConfig, ListingOptions};+ /// # use datafusion_datasource::ListingTableUrl;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap();+ /// let options = ListingOptions::new(Arc::new(ParquetFormat::default()))+ /// .with_file_extension(".parquet")+ /// .with_collect_stat(true);+ ///+ /// let config = ListingTableConfig::new(table_paths)+ /// .with_listing_options(options); // Configure file format and options+ /// ```+ pub fn with_listing_options(self, listing_options: ListingOptions) -> Self {+ // Note: This method properly sets options, but be aware that downstream+ // methods like infer_schema() and try_new() require both schema and options+ // to be set to function correctly.+ debug_assert!(+ !self.table_paths.is_empty() || cfg!(test),+ "ListingTableConfig::with_listing_options called without table_paths set. \+ Consider calling new() or new_with_multi_paths() first to establish table paths."+ );++ Self {+ options: Some(listing_options),+ ..self+ }+ }++ /// Returns a tuple of `(file_extension, optional compression_extension)`+ ///+ /// For example a path ending with blah.test.csv.gz returns `("csv", Some("gz"))`+ /// For example a path ending with blah.test.csv returns `("csv", None)`+ pub fn infer_file_extension_and_compression_type(+ path: &str,+ ) -> datafusion_common::Result<(String, Option<String>)> {+ let mut exts = path.rsplit('.');++ let split = exts.next().unwrap_or("");++ let file_compression_type = FileCompressionType::from_str(split)+ .unwrap_or(FileCompressionType::UNCOMPRESSED);++ if file_compression_type.is_compressed() {+ let split2 = exts.next().unwrap_or("");+ Ok((split2.to_string(), Some(split.to_string())))+ } else {+ Ok((split.to_string(), None))+ }+ }++ /// Infer the [`SchemaRef`] based on `table_path`s.+ ///+ /// This method infers the table schema using the first `table_path`.+ /// See [`ListingOptions::infer_schema`] for more details+ ///+ /// # Errors+ /// * if `self.options` is not set. See [`Self::with_listing_options`]+ pub async fn infer_schema(+ self,+ state: &dyn Session,+ ) -> datafusion_common::Result<Self> {+ match self.options {+ Some(options) => {+ let ListingTableConfig {+ table_paths,+ file_schema,+ options: _,+ schema_source,+ schema_adapter_factory,+ expr_adapter_factory: physical_expr_adapter_factory,+ } = self;++ let (schema, new_schema_source) = match file_schema {+ Some(schema) => (schema, schema_source), // Keep existing source if schema exists+ None => {+ if let Some(url) = table_paths.first() {+ (+ options.infer_schema(state, url).await?,+ SchemaSource::Inferred,+ )+ } else {+ (Arc::new(Schema::empty()), SchemaSource::Inferred)+ }+ }+ };++ Ok(Self {+ table_paths,+ file_schema: Some(schema),+ options: Some(options),+ schema_source: new_schema_source,+ schema_adapter_factory,+ expr_adapter_factory: physical_expr_adapter_factory,+ })+ }+ None => internal_err!("No `ListingOptions` set for inferring schema"),+ }+ }++ /// Infer the partition columns from `table_paths`.+ ///+ /// # Errors+ /// * if `self.options` is not set. See [`Self::with_listing_options`]+ pub async fn infer_partitions_from_path(+ self,+ state: &dyn Session,+ ) -> datafusion_common::Result<Self> {+ match self.options {+ Some(options) => {+ let Some(url) = self.table_paths.first() else {+ return config_err!("No table path found");+ };+ let partitions = options+ .infer_partitions(state, url)+ .await?+ .into_iter()+ .map(|col_name| {+ (+ col_name,+ DataType::Dictionary(+ Box::new(DataType::UInt16),+ Box::new(DataType::Utf8),+ ),+ )+ })+ .collect::<Vec<_>>();+ let options = options.with_table_partition_cols(partitions);+ Ok(Self {+ table_paths: self.table_paths,+ file_schema: self.file_schema,+ options: Some(options),+ schema_source: self.schema_source,+ schema_adapter_factory: self.schema_adapter_factory,+ expr_adapter_factory: self.expr_adapter_factory,+ })+ }+ None => config_err!("No `ListingOptions` set for inferring schema"),+ }+ }++ /// Set the [`SchemaAdapterFactory`] for the [`crate::ListingTable`]+ ///+ /// The schema adapter factory is used to create schema adapters that can+ /// handle schema evolution and type conversions when reading files with+ /// different schemas than the table schema.+ ///+ /// If not provided, a default schema adapter factory will be used.+ ///+ /// # Example: Custom Schema Adapter for Type Coercion+ /// ```rust+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::{ListingTableConfig, ListingOptions};+ /// # use datafusion_datasource::schema_adapter::{SchemaAdapterFactory, SchemaAdapter};+ /// # use datafusion_datasource::ListingTableUrl;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType};+ /// #+ /// # #[derive(Debug)]+ /// # struct MySchemaAdapterFactory;+ /// # impl SchemaAdapterFactory for MySchemaAdapterFactory {+ /// # fn create(&self, _projected_table_schema: SchemaRef, _file_schema: SchemaRef) -> Box<dyn SchemaAdapter> {+ /// # unimplemented!()+ /// # }+ /// # }+ /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap();+ /// # let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default()));+ /// # let table_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));+ /// let config = ListingTableConfig::new(table_paths)+ /// .with_listing_options(listing_options)+ /// .with_schema(table_schema)+ /// .with_schema_adapter_factory(Arc::new(MySchemaAdapterFactory));+ /// ```+ pub fn with_schema_adapter_factory(+ self,+ schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,+ ) -> Self {+ Self {+ schema_adapter_factory: Some(schema_adapter_factory),+ ..self+ }+ }++ /// Get the [`SchemaAdapterFactory`] for this configuration+ pub fn schema_adapter_factory(&self) -> Option<&Arc<dyn SchemaAdapterFactory>> {+ self.schema_adapter_factory.as_ref()+ }++ /// Set the [`PhysicalExprAdapterFactory`] for the [`crate::ListingTable`]+ ///+ /// The expression adapter factory is used to create physical expression adapters that can+ /// handle schema evolution and type conversions when evaluating expressions+ /// with different schemas than the table schema.+ ///+ /// If not provided, a default physical expression adapter factory will be used unless a custom+ /// `SchemaAdapterFactory` is set, in which case only the `SchemaAdapterFactory` will be used.+ ///+ /// See <https://github.com/apache/datafusion/issues/16800> for details on this transition.+ pub fn with_expr_adapter_factory(+ self,+ expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,+ ) -> Self {+ Self {+ expr_adapter_factory: Some(expr_adapter_factory),+ ..self+ }+ }+}diff --git a/datafusion/catalog-listing/src/mod.rs b/datafusion/catalog-listing/src/mod.rsindex 1322577b20..90d04b46b8 100644--- a/datafusion/catalog-listing/src/mod.rs+++ b/datafusion/catalog-listing/src/mod.rs@@ -24,4 +24,11 @@// https://github.com/apache/datafusion/issues/11143#![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]+mod config;pub mod helpers;+mod options;+mod table;++pub use config::{ListingTableConfig, SchemaSource};+pub use options::ListingOptions;+pub use table::ListingTable;diff --git a/datafusion/catalog-listing/src/options.rs b/datafusion/catalog-listing/src/options.rsnew file mode 100644index 0000000000..3cbf3573e9--- /dev/null+++ b/datafusion/catalog-listing/src/options.rs@@ -0,0 +1,411 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::datatypes::{DataType, SchemaRef};+use datafusion_catalog::Session;+use datafusion_common::plan_err;+use datafusion_datasource::file_format::FileFormat;+use datafusion_datasource::ListingTableUrl;+use datafusion_execution::config::SessionConfig;+use datafusion_expr::SortExpr;+use futures::StreamExt;+use futures::{future, TryStreamExt};+use itertools::Itertools;+use std::sync::Arc;++/// Options for creating a [`crate::ListingTable`]+#[derive(Clone, Debug)]+pub struct ListingOptions {+ /// A suffix on which files should be filtered (leave empty to+ /// keep all files on the path)+ pub file_extension: String,+ /// The file format+ pub format: Arc<dyn FileFormat>,+ /// The expected partition column names in the folder structure.+ /// See [Self::with_table_partition_cols] for details+ pub table_partition_cols: Vec<(String, DataType)>,+ /// Set true to try to guess statistics from the files.+ /// This can add a lot of overhead as it will usually require files+ /// to be opened and at least partially parsed.+ pub collect_stat: bool,+ /// Group files to avoid that the number of partitions exceeds+ /// this limit+ pub target_partitions: usize,+ /// Optional pre-known sort order(s). Must be `SortExpr`s.+ ///+ /// DataFusion may take advantage of this ordering to omit sorts+ /// or use more efficient algorithms. Currently sortedness must be+ /// provided if it is known by some external mechanism, but may in+ /// the future be automatically determined, for example using+ /// parquet metadata.+ ///+ /// See <https://github.com/apache/datafusion/issues/4177>+ ///+ /// NOTE: This attribute stores all equivalent orderings (the outer `Vec`)+ /// where each ordering consists of an individual lexicographic+ /// ordering (encapsulated by a `Vec<Expr>`). If there aren't+ /// multiple equivalent orderings, the outer `Vec` will have a+ /// single element.+ pub file_sort_order: Vec<Vec<SortExpr>>,+}++impl ListingOptions {+ /// Creates an options instance with the given format+ /// Default values:+ /// - use default file extension filter+ /// - no input partition to discover+ /// - one target partition+ /// - do not collect statistics+ pub fn new(format: Arc<dyn FileFormat>) -> Self {+ Self {+ file_extension: format.get_ext(),+ format,+ table_partition_cols: vec![],+ collect_stat: false,+ target_partitions: 1,+ file_sort_order: vec![],+ }+ }++ /// Set options from [`SessionConfig`] and returns self.+ ///+ /// Currently this sets `target_partitions` and `collect_stat`+ /// but if more options are added in the future that need to be coordinated+ /// they will be synchronized through this method.+ pub fn with_session_config_options(mut self, config: &SessionConfig) -> Self {+ self = self.with_target_partitions(config.target_partitions());+ self = self.with_collect_stat(config.collect_statistics());+ self+ }++ /// Set file extension on [`ListingOptions`] and returns self.+ ///+ /// # Example+ /// ```+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::ListingOptions;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ ///+ /// let listing_options = ListingOptions::new(Arc::new(+ /// ParquetFormat::default()+ /// ))+ /// .with_file_extension(".parquet");+ ///+ /// assert_eq!(listing_options.file_extension, ".parquet");+ /// ```+ pub fn with_file_extension(mut self, file_extension: impl Into<String>) -> Self {+ self.file_extension = file_extension.into();+ self+ }++ /// Optionally set file extension on [`ListingOptions`] and returns self.+ ///+ /// If `file_extension` is `None`, the file extension will not be changed+ ///+ /// # Example+ /// ```+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::ListingOptions;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ ///+ /// let extension = Some(".parquet");+ /// let listing_options = ListingOptions::new(Arc::new(+ /// ParquetFormat::default()+ /// ))+ /// .with_file_extension_opt(extension);+ ///+ /// assert_eq!(listing_options.file_extension, ".parquet");+ /// ```+ pub fn with_file_extension_opt<S>(mut self, file_extension: Option<S>) -> Self+ where+ S: Into<String>,+ {+ if let Some(file_extension) = file_extension {+ self.file_extension = file_extension.into();+ }+ self+ }++ /// Set `table partition columns` on [`ListingOptions`] and returns self.+ ///+ /// "partition columns," used to support [Hive Partitioning], are+ /// columns added to the data that is read, based on the folder+ /// structure where the data resides.+ ///+ /// For example, give the following files in your filesystem:+ ///+ /// ```text+ /// /mnt/nyctaxi/year=2022/month=01/tripdata.parquet+ /// /mnt/nyctaxi/year=2021/month=12/tripdata.parquet+ /// /mnt/nyctaxi/year=2021/month=11/tripdata.parquet+ /// ```+ ///+ /// A [`crate::ListingTable`] created at `/mnt/nyctaxi/` with partition+ /// columns "year" and "month" will include new `year` and `month`+ /// columns while reading the files. The `year` column would have+ /// value `2022` and the `month` column would have value `01` for+ /// the rows read from+ /// `/mnt/nyctaxi/year=2022/month=01/tripdata.parquet`+ ///+ ///# Notes+ ///+ /// - If only one level (e.g. `year` in the example above) is+ /// specified, the other levels are ignored but the files are+ /// still read.+ ///+ /// - Files that don't follow this partitioning scheme will be+ /// ignored.+ ///+ /// - Since the columns have the same value for all rows read from+ /// each individual file (such as dates), they are typically+ /// dictionary encoded for efficiency. You may use+ /// [`wrap_partition_type_in_dict`] to request a+ /// dictionary-encoded type.+ ///+ /// - The partition columns are solely extracted from the file path. Especially they are NOT part of the parquet files itself.+ ///+ /// # Example+ ///+ /// ```+ /// # use std::sync::Arc;+ /// # use arrow::datatypes::DataType;+ /// # use datafusion_expr::col;+ /// # use datafusion_catalog_listing::ListingOptions;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ ///+ /// // listing options for files with paths such as `/mnt/data/col_a=x/col_b=y/data.parquet`+ /// // `col_a` and `col_b` will be included in the data read from those files+ /// let listing_options = ListingOptions::new(Arc::new(+ /// ParquetFormat::default()+ /// ))+ /// .with_table_partition_cols(vec![("col_a".to_string(), DataType::Utf8),+ /// ("col_b".to_string(), DataType::Utf8)]);+ ///+ /// assert_eq!(listing_options.table_partition_cols, vec![("col_a".to_string(), DataType::Utf8),+ /// ("col_b".to_string(), DataType::Utf8)]);+ /// ```+ ///+ /// [Hive Partitioning]: https://docs.cloudera.com/HDPDocuments/HDP2/HDP-2.1.3/bk_system-admin-guide/content/hive_partitioned_tables.html+ /// [`wrap_partition_type_in_dict`]: datafusion_datasource::file_scan_config::wrap_partition_type_in_dict+ pub fn with_table_partition_cols(+ mut self,+ table_partition_cols: Vec<(String, DataType)>,+ ) -> Self {+ self.table_partition_cols = table_partition_cols;+ self+ }++ /// Set stat collection on [`ListingOptions`] and returns self.+ ///+ /// ```+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::ListingOptions;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ ///+ /// let listing_options = ListingOptions::new(Arc::new(+ /// ParquetFormat::default()+ /// ))+ /// .with_collect_stat(true);+ ///+ /// assert_eq!(listing_options.collect_stat, true);+ /// ```+ pub fn with_collect_stat(mut self, collect_stat: bool) -> Self {+ self.collect_stat = collect_stat;+ self+ }++ /// Set number of target partitions on [`ListingOptions`] and returns self.+ ///+ /// ```+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::ListingOptions;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ ///+ /// let listing_options = ListingOptions::new(Arc::new(+ /// ParquetFormat::default()+ /// ))+ /// .with_target_partitions(8);+ ///+ /// assert_eq!(listing_options.target_partitions, 8);+ /// ```+ pub fn with_target_partitions(mut self, target_partitions: usize) -> Self {+ self.target_partitions = target_partitions;+ self+ }++ /// Set file sort order on [`ListingOptions`] and returns self.+ ///+ /// ```+ /// # use std::sync::Arc;+ /// # use datafusion_expr::col;+ /// # use datafusion_catalog_listing::ListingOptions;+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ ///+ /// // Tell datafusion that the files are sorted by column "a"+ /// let file_sort_order = vec![vec![+ /// col("a").sort(true, true)+ /// ]];+ ///+ /// let listing_options = ListingOptions::new(Arc::new(+ /// ParquetFormat::default()+ /// ))+ /// .with_file_sort_order(file_sort_order.clone());+ ///+ /// assert_eq!(listing_options.file_sort_order, file_sort_order);+ /// ```+ pub fn with_file_sort_order(mut self, file_sort_order: Vec<Vec<SortExpr>>) -> Self {+ self.file_sort_order = file_sort_order;+ self+ }++ /// Infer the schema of the files at the given path on the provided object store.+ ///+ /// If the table_path contains one or more files (i.e. it is a directory /+ /// prefix of files) their schema is merged by calling [`FileFormat::infer_schema`]+ ///+ /// Note: The inferred schema does not include any partitioning columns.+ ///+ /// This method is called as part of creating a [`crate::ListingTable`].+ pub async fn infer_schema<'a>(+ &'a self,+ state: &dyn Session,+ table_path: &'a ListingTableUrl,+ ) -> datafusion_common::Result<SchemaRef> {+ let store = state.runtime_env().object_store(table_path)?;++ let files: Vec<_> = table_path+ .list_all_files(state, store.as_ref(), &self.file_extension)+ .await?+ // Empty files cannot affect schema but may throw when trying to read for it+ .try_filter(|object_meta| future::ready(object_meta.size > 0))+ .try_collect()+ .await?;++ let schema = self.format.infer_schema(state, &store, &files).await?;++ Ok(schema)+ }++ /// Infers the partition columns stored in `LOCATION` and compares+ /// them with the columns provided in `PARTITIONED BY` to help prevent+ /// accidental corrupts of partitioned tables.+ ///+ /// Allows specifying partial partitions.+ pub async fn validate_partitions(+ &self,+ state: &dyn Session,+ table_path: &ListingTableUrl,+ ) -> datafusion_common::Result<()> {+ if self.table_partition_cols.is_empty() {+ return Ok(());+ }++ if !table_path.is_collection() {+ return plan_err!(+ "Can't create a partitioned table backed by a single file, \+ perhaps the URL is missing a trailing slash?"+ );+ }++ let inferred = self.infer_partitions(state, table_path).await?;++ // no partitioned files found on disk+ if inferred.is_empty() {+ return Ok(());+ }++ let table_partition_names = self+ .table_partition_cols+ .iter()+ .map(|(col_name, _)| col_name.clone())+ .collect_vec();++ if inferred.len() < table_partition_names.len() {+ return plan_err!(+ "Inferred partitions to be {:?}, but got {:?}",+ inferred,+ table_partition_names+ );+ }++ // match prefix to allow creating tables with partial partitions+ for (idx, col) in table_partition_names.iter().enumerate() {+ if &inferred[idx] != col {+ return plan_err!(+ "Inferred partitions to be {:?}, but got {:?}",+ inferred,+ table_partition_names+ );+ }+ }++ Ok(())+ }++ /// Infer the partitioning at the given path on the provided object store.+ /// For performance reasons, it doesn't read all the files on disk+ /// and therefore may fail to detect invalid partitioning.+ pub async fn infer_partitions(+ &self,+ state: &dyn Session,+ table_path: &ListingTableUrl,+ ) -> datafusion_common::Result<Vec<String>> {+ let store = state.runtime_env().object_store(table_path)?;++ // only use 10 files for inference+ // This can fail to detect inconsistent partition keys+ // A DFS traversal approach of the store can help here+ let files: Vec<_> = table_path+ .list_all_files(state, store.as_ref(), &self.file_extension)+ .await?+ .take(10)+ .try_collect()+ .await?;++ let stripped_path_parts = files.iter().map(|file| {+ table_path+ .strip_prefix(&file.location)+ .unwrap()+ .collect_vec()+ });++ let partition_keys = stripped_path_parts+ .map(|path_parts| {+ path_parts+ .into_iter()+ .rev()+ .skip(1) // get parents only; skip the file itself+ .rev()+ // Partitions are expected to follow the format "column_name=value", so we+ // should ignore any path part that cannot be parsed into the expected format+ .filter(|s| s.contains('='))+ .map(|s| s.split('=').take(1).collect())+ .collect_vec()+ })+ .collect_vec();++ match partition_keys.into_iter().all_equal_value() {+ Ok(v) => Ok(v),+ Err(None) => Ok(vec![]),+ Err(Some(diff)) => {+ let mut sorted_diff = [diff.0, diff.1];+ sorted_diff.sort();+ plan_err!("Found mixed partition values on disk {:?}", sorted_diff)+ }+ }+ }+}diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsnew file mode 100644index 0000000000..e9ac1bf097--- /dev/null+++ b/datafusion/catalog-listing/src/table.rs@@ -0,0 +1,788 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use crate::config::SchemaSource;+use crate::helpers::{expr_applicable_for_cols, pruned_partition_list};+use crate::{ListingOptions, ListingTableConfig};+use arrow::datatypes::{Field, Schema, SchemaBuilder, SchemaRef};+use async_trait::async_trait;+use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};+use datafusion_common::stats::Precision;+use datafusion_common::{+ internal_datafusion_err, plan_err, project_schema, Constraints, DataFusionError,+ SchemaExt, Statistics,+};+use datafusion_datasource::file::FileSource;+use datafusion_datasource::file_groups::FileGroup;+use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};+use datafusion_datasource::file_sink_config::FileSinkConfig;+use datafusion_datasource::schema_adapter::{+ DefaultSchemaAdapterFactory, SchemaAdapter, SchemaAdapterFactory,+};+use datafusion_datasource::{+ compute_all_files_statistics, ListingTableUrl, PartitionedFile,+};+use datafusion_execution::cache::cache_manager::FileStatisticsCache;+use datafusion_execution::cache::cache_unit::DefaultFileStatisticsCache;+use datafusion_expr::dml::InsertOp;+use datafusion_expr::execution_props::ExecutionProps;+use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType};+use datafusion_physical_expr::create_lex_ordering;+use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;+use datafusion_physical_expr_common::sort_expr::LexOrdering;+use datafusion_physical_plan::empty::EmptyExec;+use datafusion_physical_plan::ExecutionPlan;+use futures::{future, stream, Stream, StreamExt, TryStreamExt};+use object_store::ObjectStore;+use std::any::Any;+use std::collections::HashMap;+use std::sync::Arc;++/// Built in [`TableProvider`] that reads data from one or more files as a single table.+///+/// The files are read using an [`ObjectStore`] instance, for example from+/// local files or objects from AWS S3.+///+/// # Features:+/// * Reading multiple files as a single table+/// * Hive style partitioning (e.g., directories named `date=2024-06-01`)+/// * Merges schemas from files with compatible but not identical schemas (see [`ListingTableConfig::file_schema`])+/// * `limit`, `filter` and `projection` pushdown for formats that support it (e.g.,+/// Parquet)+/// * Statistics collection and pruning based on file metadata+/// * Pre-existing sort order (see [`ListingOptions::file_sort_order`])+/// * Metadata caching to speed up repeated queries (see [`FileMetadataCache`])+/// * Statistics caching (see [`FileStatisticsCache`])+///+/// [`FileMetadataCache`]: datafusion_execution::cache::cache_manager::FileMetadataCache+///+/// # Reading Directories and Hive Style Partitioning+///+/// For example, given the `table1` directory (or object store prefix)+///+/// ```text+/// table1+/// ├── file1.parquet+/// └── file2.parquet+/// ```+///+/// A `ListingTable` would read the files `file1.parquet` and `file2.parquet` as+/// a single table, merging the schemas if the files have compatible but not+/// identical schemas.+///+/// Given the `table2` directory (or object store prefix)+///+/// ```text+/// table2+/// ├── date=2024-06-01+/// │ ├── file3.parquet+/// │ └── file4.parquet+/// └── date=2024-06-02+/// └── file5.parquet+/// ```+///+/// A `ListingTable` would read the files `file3.parquet`, `file4.parquet`, and+/// `file5.parquet` as a single table, again merging schemas if necessary.+///+/// Given the hive style partitioning structure (e.g,. directories named+/// `date=2024-06-01` and `date=2026-06-02`), `ListingTable` also adds a `date`+/// column when reading the table:+/// * The files in `table2/date=2024-06-01` will have the value `2024-06-01`+/// * The files in `table2/date=2024-06-02` will have the value `2024-06-02`.+///+/// If the query has a predicate like `WHERE date = '2024-06-01'`+/// only the corresponding directory will be read.+///+/// # See Also+///+/// 1. [`ListingTableConfig`]: Configuration options+/// 1. [`DataSourceExec`]: `ExecutionPlan` used by `ListingTable`+///+/// [`DataSourceExec`]: datafusion_datasource::source::DataSourceExec+///+/// # Caching Metadata+///+/// Some formats, such as Parquet, use the `FileMetadataCache` to cache file+/// metadata that is needed to execute but expensive to read, such as row+/// groups and statistics. The cache is scoped to the `SessionContext` and can+/// be configured via the [runtime config options].+///+/// [runtime config options]: https://datafusion.apache.org/user-guide/configs.html#runtime-configuration-settings+///+/// # Example: Read a directory of parquet files using a [`ListingTable`]+///+/// ```no_run+/// # use datafusion_common::Result;+/// # use std::sync::Arc;+/// # use datafusion_catalog::TableProvider;+/// # use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};+/// # use datafusion_datasource::ListingTableUrl;+/// # use datafusion_datasource_parquet::file_format::ParquetFormat;/// #+/// # use datafusion_catalog::Session;+/// async fn get_listing_table(session: &dyn Session) -> Result<Arc<dyn TableProvider>> {+/// let table_path = "/path/to/parquet";+///+/// // Parse the path+/// let table_path = ListingTableUrl::parse(table_path)?;+///+/// // Create default parquet options+/// let file_format = ParquetFormat::new();+/// let listing_options = ListingOptions::new(Arc::new(file_format))+/// .with_file_extension(".parquet");+///+/// // Resolve the schema+/// let resolved_schema = listing_options+/// .infer_schema(session, &table_path)+/// .await?;+///+/// let config = ListingTableConfig::new(table_path)+/// .with_listing_options(listing_options)+/// .with_schema(resolved_schema);+///+/// // Create a new TableProvider+/// let provider = Arc::new(ListingTable::try_new(config)?);+///+/// # Ok(provider)+/// # }+/// ```+#[derive(Debug, Clone)]+pub struct ListingTable {+ table_paths: Vec<ListingTableUrl>,+ /// `file_schema` contains only the columns physically stored in the data files themselves.+ /// - Represents the actual fields found in files like Parquet, CSV, etc.+ /// - Used when reading the raw data from files+ file_schema: SchemaRef,+ /// `table_schema` combines `file_schema` + partition columns+ /// - Partition columns are derived from directory paths (not stored in files)+ /// - These are columns like "year=2022/month=01" in paths like `/data/year=2022/month=01/file.parquet`+ table_schema: SchemaRef,+ /// Indicates how the schema was derived (inferred or explicitly specified)+ schema_source: SchemaSource,+ /// Options used to configure the listing table such as the file format+ /// and partitioning information+ options: ListingOptions,+ /// The SQL definition for this table, if any+ definition: Option<String>,+ /// Cache for collected file statistics+ collected_statistics: FileStatisticsCache,+ /// Constraints applied to this table+ constraints: Constraints,+ /// Column default expressions for columns that are not physically present in the data files+ column_defaults: HashMap<String, Expr>,+ /// Optional [`SchemaAdapterFactory`] for creating schema adapters+ schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,+ /// Optional [`PhysicalExprAdapterFactory`] for creating physical expression adapters+ expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,+}++impl ListingTable {+ /// Create new [`ListingTable`]+ ///+ /// See documentation and example on [`ListingTable`] and [`ListingTableConfig`]+ pub fn try_new(config: ListingTableConfig) -> datafusion_common::Result<Self> {+ // Extract schema_source before moving other parts of the config+ let schema_source = config.schema_source();++ let file_schema = config+ .file_schema+ .ok_or_else(|| internal_datafusion_err!("No schema provided."))?;++ let options = config+ .options+ .ok_or_else(|| internal_datafusion_err!("No ListingOptions provided"))?;++ // Add the partition columns to the file schema+ let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned());+ for (part_col_name, part_col_type) in &options.table_partition_cols {+ builder.push(Field::new(part_col_name, part_col_type.clone(), false));+ }++ let table_schema = Arc::new(+ builder+ .finish()+ .with_metadata(file_schema.metadata().clone()),+ );++ let table = Self {+ table_paths: config.table_paths,+ file_schema,+ table_schema,+ schema_source,+ options,+ definition: None,+ collected_statistics: Arc::new(DefaultFileStatisticsCache::default()),+ constraints: Constraints::default(),+ column_defaults: HashMap::new(),+ schema_adapter_factory: config.schema_adapter_factory,+ expr_adapter_factory: config.expr_adapter_factory,+ };++ Ok(table)+ }++ /// Assign constraints+ pub fn with_constraints(mut self, constraints: Constraints) -> Self {+ self.constraints = constraints;+ self+ }++ /// Assign column defaults+ pub fn with_column_defaults(+ mut self,+ column_defaults: HashMap<String, Expr>,+ ) -> Self {+ self.column_defaults = column_defaults;+ self+ }++ /// Set the [`FileStatisticsCache`] used to cache parquet file statistics.+ ///+ /// Setting a statistics cache on the `SessionContext` can avoid refetching statistics+ /// multiple times in the same session.+ ///+ /// If `None`, creates a new [`DefaultFileStatisticsCache`] scoped to this query.+ pub fn with_cache(mut self, cache: Option<FileStatisticsCache>) -> Self {+ self.collected_statistics =+ cache.unwrap_or_else(|| Arc::new(DefaultFileStatisticsCache::default()));+ self+ }++ /// Specify the SQL definition for this table, if any+ pub fn with_definition(mut self, definition: Option<String>) -> Self {+ self.definition = definition;+ self+ }++ /// Get paths ref+ pub fn table_paths(&self) -> &Vec<ListingTableUrl> {+ &self.table_paths+ }++ /// Get options ref+ pub fn options(&self) -> &ListingOptions {+ &self.options+ }++ /// Get the schema source+ pub fn schema_source(&self) -> SchemaSource {+ self.schema_source+ }++ /// Set the [`SchemaAdapterFactory`] for this [`ListingTable`]+ ///+ /// The schema adapter factory is used to create schema adapters that can+ /// handle schema evolution and type conversions when reading files with+ /// different schemas than the table schema.+ ///+ /// # Example: Adding Schema Evolution Support+ /// ```rust+ /// # use std::sync::Arc;+ /// # use datafusion_catalog_listing::{ListingTable, ListingTableConfig, ListingOptions};+ /// # use datafusion_datasource::ListingTableUrl;+ /// # use datafusion_datasource::schema_adapter::{DefaultSchemaAdapterFactory, SchemaAdapter};+ /// # use datafusion_datasource_parquet::file_format::ParquetFormat;+ /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType};+ /// # let table_path = ListingTableUrl::parse("file:///path/to/data").unwrap();+ /// # let options = ListingOptions::new(Arc::new(ParquetFormat::default()));+ /// # let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));+ /// # let config = ListingTableConfig::new(table_path).with_listing_options(options).with_schema(schema);+ /// # let table = ListingTable::try_new(config).unwrap();+ /// let table_with_evolution = table+ /// .with_schema_adapter_factory(Arc::new(DefaultSchemaAdapterFactory));+ /// ```+ /// See [`ListingTableConfig::with_schema_adapter_factory`] for an example of custom SchemaAdapterFactory.+ pub fn with_schema_adapter_factory(+ self,+ schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,+ ) -> Self {+ Self {+ schema_adapter_factory: Some(schema_adapter_factory),+ ..self+ }+ }++ /// Get the [`SchemaAdapterFactory`] for this table+ pub fn schema_adapter_factory(&self) -> Option<&Arc<dyn SchemaAdapterFactory>> {+ self.schema_adapter_factory.as_ref()+ }++ /// Creates a schema adapter for mapping between file and table schemas+ ///+ /// Uses the configured schema adapter factory if available, otherwise falls back+ /// to the default implementation.+ fn create_schema_adapter(&self) -> Box<dyn SchemaAdapter> {+ let table_schema = self.schema();+ match &self.schema_adapter_factory {+ Some(factory) => {+ factory.create_with_projected_schema(Arc::clone(&table_schema))+ }+ None => DefaultSchemaAdapterFactory::from_schema(Arc::clone(&table_schema)),+ }+ }++ /// Creates a file source and applies schema adapter factory if available+ fn create_file_source_with_schema_adapter(+ &self,+ ) -> datafusion_common::Result<Arc<dyn FileSource>> {+ let mut source = self.options.format.file_source();+ // Apply schema adapter to source if available+ //+ // The source will use this SchemaAdapter to adapt data batches as they flow up the plan.+ // Note: ListingTable also creates a SchemaAdapter in `scan()` but that is only used to adapt collected statistics.+ if let Some(factory) = &self.schema_adapter_factory {+ source = source.with_schema_adapter_factory(Arc::clone(factory))?;+ }+ Ok(source)+ }++ /// If file_sort_order is specified, creates the appropriate physical expressions+ pub fn try_create_output_ordering(+ &self,+ execution_props: &ExecutionProps,+ ) -> datafusion_common::Result<Vec<LexOrdering>> {+ create_lex_ordering(+ &self.table_schema,+ &self.options.file_sort_order,+ execution_props,+ )+ }+}++// Expressions can be used for partition pruning if they can be evaluated using+// only the partition columns and there are partition columns.+fn can_be_evaluated_for_partition_pruning(+ partition_column_names: &[&str],+ expr: &Expr,+) -> bool {+ !partition_column_names.is_empty()+ && expr_applicable_for_cols(partition_column_names, expr)+}++#[async_trait]+impl TableProvider for ListingTable {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.table_schema)+ }++ fn constraints(&self) -> Option<&Constraints> {+ Some(&self.constraints)+ }++ fn table_type(&self) -> TableType {+ TableType::Base+ }++ async fn scan(+ &self,+ state: &dyn Session,+ projection: Option<&Vec<usize>>,+ filters: &[Expr],+ limit: Option<usize>,+ ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {+ let options = ScanArgs::default()+ .with_projection(projection.map(|p| p.as_slice()))+ .with_filters(Some(filters))+ .with_limit(limit);+ Ok(self.scan_with_args(state, options).await?.into_inner())+ }++ async fn scan_with_args<'a>(+ &self,+ state: &dyn Session,+ args: ScanArgs<'a>,+ ) -> datafusion_common::Result<ScanResult> {+ let projection = args.projection().map(|p| p.to_vec());+ let filters = args.filters().map(|f| f.to_vec()).unwrap_or_default();+ let limit = args.limit();++ // extract types of partition columns+ let table_partition_cols = self+ .options+ .table_partition_cols+ .iter()+ .map(|col| Ok(self.table_schema.field_with_name(&col.0)?.clone()))+ .collect::<datafusion_common::Result<Vec<_>>>()?;++ let table_partition_col_names = table_partition_cols+ .iter()+ .map(|field| field.name().as_str())+ .collect::<Vec<_>>();++ // If the filters can be resolved using only partition cols, there is no need to+ // pushdown it to TableScan, otherwise, `unhandled` pruning predicates will be generated+ let (partition_filters, filters): (Vec<_>, Vec<_>) =+ filters.iter().cloned().partition(|filter| {+ can_be_evaluated_for_partition_pruning(&table_partition_col_names, filter)+ });++ // We should not limit the number of partitioned files to scan if there are filters and limit+ // at the same time. This is because the limit should be applied after the filters are applied.+ let statistic_file_limit = if filters.is_empty() { limit } else { None };++ let (mut partitioned_file_lists, statistics) = self+ .list_files_for_scan(state, &partition_filters, statistic_file_limit)+ .await?;++ // if no files need to be read, return an `EmptyExec`+ if partitioned_file_lists.is_empty() {+ let projected_schema = project_schema(&self.schema(), projection.as_ref())?;+ return Ok(ScanResult::new(Arc::new(EmptyExec::new(projected_schema))));+ }++ let output_ordering = self.try_create_output_ordering(state.execution_props())?;+ match state+ .config_options()+ .execution+ .split_file_groups_by_statistics+ .then(|| {+ output_ordering.first().map(|output_ordering| {+ FileScanConfig::split_groups_by_statistics_with_target_partitions(+ &self.table_schema,+ &partitioned_file_lists,+ output_ordering,+ self.options.target_partitions,+ )+ })+ })+ .flatten()+ {+ Some(Err(e)) => log::debug!("failed to split file groups by statistics: {e}"),+ Some(Ok(new_groups)) => {+ if new_groups.len() <= self.options.target_partitions {+ partitioned_file_lists = new_groups;+ } else {+ log::debug!("attempted to split file groups by statistics, but there were more file groups than target_partitions; falling back to unordered")+ }+ }+ None => {} // no ordering required+ };++ let Some(object_store_url) =+ self.table_paths.first().map(ListingTableUrl::object_store)+ else {+ return Ok(ScanResult::new(Arc::new(EmptyExec::new(Arc::new(+ Schema::empty(),+ )))));+ };++ let file_source = self.create_file_source_with_schema_adapter()?;++ // create the execution plan+ let plan = self+ .options+ .format+ .create_physical_plan(+ state,+ FileScanConfigBuilder::new(+ object_store_url,+ Arc::clone(&self.file_schema),+ file_source,+ )+ .with_file_groups(partitioned_file_lists)+ .with_constraints(self.constraints.clone())+ .with_statistics(statistics)+ .with_projection(projection)+ .with_limit(limit)+ .with_output_ordering(output_ordering)+ .with_table_partition_cols(table_partition_cols)+ .with_expr_adapter(self.expr_adapter_factory.clone())+ .build(),+ )+ .await?;++ Ok(ScanResult::new(plan))+ }++ fn supports_filters_pushdown(+ &self,+ filters: &[&Expr],+ ) -> datafusion_common::Result<Vec<TableProviderFilterPushDown>> {+ let partition_column_names = self+ .options+ .table_partition_cols+ .iter()+ .map(|col| col.0.as_str())+ .collect::<Vec<_>>();+ filters+ .iter()+ .map(|filter| {+ if can_be_evaluated_for_partition_pruning(&partition_column_names, filter)+ {+ // if filter can be handled by partition pruning, it is exact+ return Ok(TableProviderFilterPushDown::Exact);+ }++ Ok(TableProviderFilterPushDown::Inexact)+ })+ .collect()+ }++ fn get_table_definition(&self) -> Option<&str> {+ self.definition.as_deref()+ }++ async fn insert_into(+ &self,+ state: &dyn Session,+ input: Arc<dyn ExecutionPlan>,+ insert_op: InsertOp,+ ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {+ // Check that the schema of the plan matches the schema of this table.+ self.schema()+ .logically_equivalent_names_and_types(&input.schema())?;++ let table_path = &self.table_paths()[0];+ if !table_path.is_collection() {+ return plan_err!(+ "Inserting into a ListingTable backed by a single file is not supported, URL is possibly missing a trailing `/`. \+ To append to an existing file use StreamTable, e.g. by using CREATE UNBOUNDED EXTERNAL TABLE"+ );+ }++ // Get the object store for the table path.+ let store = state.runtime_env().object_store(table_path)?;++ let file_list_stream = pruned_partition_list(+ state,+ store.as_ref(),+ table_path,+ &[],+ &self.options.file_extension,+ &self.options.table_partition_cols,+ )+ .await?;++ let file_group = file_list_stream.try_collect::<Vec<_>>().await?.into();+ let keep_partition_by_columns =+ state.config_options().execution.keep_partition_by_columns;++ // Sink related option, apart from format+ let config = FileSinkConfig {+ original_url: String::default(),+ object_store_url: self.table_paths()[0].object_store(),+ table_paths: self.table_paths().clone(),+ file_group,+ output_schema: self.schema(),+ table_partition_cols: self.options.table_partition_cols.clone(),+ insert_op,+ keep_partition_by_columns,+ file_extension: self.options().format.get_ext(),+ };++ let orderings = self.try_create_output_ordering(state.execution_props())?;+ // It is sufficient to pass only one of the equivalent orderings:+ let order_requirements = orderings.into_iter().next().map(Into::into);++ self.options()+ .format+ .create_writer_physical_plan(input, state, config, order_requirements)+ .await+ }++ fn get_column_default(&self, column: &str) -> Option<&Expr> {+ self.column_defaults.get(column)+ }+}++impl ListingTable {+ /// Get the list of files for a scan as well as the file level statistics.+ /// The list is grouped to let the execution plan know how the files should+ /// be distributed to different threads / executors.+ pub async fn list_files_for_scan<'a>(+ &'a self,+ ctx: &'a dyn Session,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> datafusion_common::Result<(Vec<FileGroup>, Statistics)> {+ let store = if let Some(url) = self.table_paths.first() {+ ctx.runtime_env().object_store(url)?+ } else {+ return Ok((vec![], Statistics::new_unknown(&self.file_schema)));+ };+ // list files (with partitions)+ let file_list = future::try_join_all(self.table_paths.iter().map(|table_path| {+ pruned_partition_list(+ ctx,+ store.as_ref(),+ table_path,+ filters,+ &self.options.file_extension,+ &self.options.table_partition_cols,+ )+ }))+ .await?;+ let meta_fetch_concurrency =+ ctx.config_options().execution.meta_fetch_concurrency;+ let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency);+ // collect the statistics if required by the config+ let files = file_list+ .map(|part_file| async {+ let part_file = part_file?;+ let statistics = if self.options.collect_stat {+ self.do_collect_statistics(ctx, &store, &part_file).await?+ } else {+ Arc::new(Statistics::new_unknown(&self.file_schema))+ };+ Ok(part_file.with_statistics(statistics))+ })+ .boxed()+ .buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency);++ let (file_group, inexact_stats) =+ get_files_with_limit(files, limit, self.options.collect_stat).await?;++ let file_groups = file_group.split_files(self.options.target_partitions);+ let (mut file_groups, mut stats) = compute_all_files_statistics(+ file_groups,+ self.schema(),+ self.options.collect_stat,+ inexact_stats,+ )?;++ let schema_adapter = self.create_schema_adapter();+ let (schema_mapper, _) = schema_adapter.map_schema(self.file_schema.as_ref())?;++ stats.column_statistics =+ schema_mapper.map_column_statistics(&stats.column_statistics)?;+ file_groups.iter_mut().try_for_each(|file_group| {+ if let Some(stat) = file_group.statistics_mut() {+ stat.column_statistics =+ schema_mapper.map_column_statistics(&stat.column_statistics)?;+ }+ Ok::<_, DataFusionError>(())+ })?;+ Ok((file_groups, stats))+ }++ /// Collects statistics for a given partitioned file.+ ///+ /// This method first checks if the statistics for the given file are already cached.+ /// If they are, it returns the cached statistics.+ /// If they are not, it infers the statistics from the file and stores them in the cache.+ async fn do_collect_statistics(+ &self,+ ctx: &dyn Session,+ store: &Arc<dyn ObjectStore>,+ part_file: &PartitionedFile,+ ) -> datafusion_common::Result<Arc<Statistics>> {+ match self+ .collected_statistics+ .get_with_extra(&part_file.object_meta.location, &part_file.object_meta)+ {+ Some(statistics) => Ok(statistics),+ None => {+ let statistics = self+ .options+ .format+ .infer_stats(+ ctx,+ store,+ Arc::clone(&self.file_schema),+ &part_file.object_meta,+ )+ .await?;+ let statistics = Arc::new(statistics);+ self.collected_statistics.put_with_extra(+ &part_file.object_meta.location,+ Arc::clone(&statistics),+ &part_file.object_meta,+ );+ Ok(statistics)+ }+ }+ }+}++/// Processes a stream of partitioned files and returns a `FileGroup` containing the files.+///+/// This function collects files from the provided stream until either:+/// 1. The stream is exhausted+/// 2. The accumulated number of rows exceeds the provided `limit` (if specified)+///+/// # Arguments+/// * `files` - A stream of `Result<PartitionedFile>` items to process+/// * `limit` - An optional row count limit. If provided, the function will stop collecting files+/// once the accumulated number of rows exceeds this limit+/// * `collect_stats` - Whether to collect and accumulate statistics from the files+///+/// # Returns+/// A `Result` containing a `FileGroup` with the collected files+/// and a boolean indicating whether the statistics are inexact.+///+/// # Note+/// The function will continue processing files if statistics are not available or if the+/// limit is not provided. If `collect_stats` is false, statistics won't be accumulated+/// but files will still be collected.+async fn get_files_with_limit(+ files: impl Stream<Item = datafusion_common::Result<PartitionedFile>>,+ limit: Option<usize>,+ collect_stats: bool,+) -> datafusion_common::Result<(FileGroup, bool)> {+ let mut file_group = FileGroup::default();+ // Fusing the stream allows us to call next safely even once it is finished.+ let mut all_files = Box::pin(files.fuse());+ enum ProcessingState {+ ReadingFiles,+ ReachedLimit,+ }++ let mut state = ProcessingState::ReadingFiles;+ let mut num_rows = Precision::Absent;++ while let Some(file_result) = all_files.next().await {+ // Early exit if we've already reached our limit+ if matches!(state, ProcessingState::ReachedLimit) {+ break;+ }++ let file = file_result?;++ // Update file statistics regardless of state+ if collect_stats {+ if let Some(file_stats) = &file.statistics {+ num_rows = if file_group.is_empty() {+ // For the first file, just take its row count+ file_stats.num_rows+ } else {+ // For subsequent files, accumulate the counts+ num_rows.add(&file_stats.num_rows)+ };+ }+ }++ // Always add the file to our group+ file_group.push(file);++ // Check if we've hit the limit (if one was specified)+ if let Some(limit) = limit {+ if let Precision::Exact(row_count) = num_rows {+ if row_count > limit {+ state = ProcessingState::ReachedLimit;+ }+ }+ }+ }+ // If we still have files in the stream, it means that the limit kicked+ // in, and the statistic could have been different had we processed the+ // files in a different order.+ let inexact_stats = all_files.next().await.is_some();+ Ok((file_group, inexact_stats))+}diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rsindex b30d53e586..256a11ba69 100644--- a/datafusion/core/src/datasource/dynamic_file.rs+++ b/datafusion/core/src/datasource/dynamic_file.rs@@ -20,6 +20,7 @@use std::sync::Arc;+use crate::datasource::listing::ListingTableConfigExt;use crate::datasource::listing::{ListingTable, ListingTableConfig, ListingTableUrl};use crate::datasource::TableProvider;use crate::error::Result;diff --git a/datafusion/core/src/datasource/listing/mod.rs b/datafusion/core/src/datasource/listing/mod.rsindex a58db55bcc..c206566a65 100644--- a/datafusion/core/src/datasource/listing/mod.rs+++ b/datafusion/core/src/datasource/listing/mod.rs@@ -20,7 +20,8 @@mod table;pub use datafusion_catalog_listing::helpers;+pub use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};pub use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile, PartitionedFileStream,};-pub use table::{ListingOptions, ListingTable, ListingTableConfig};+pub use table::ListingTableConfigExt;diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 4ffb6d4186..3333b70676 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -15,226 +15,42 @@// specific language governing permissions and limitations// under the License.-//! The table implementation.--use super::{- helpers::{expr_applicable_for_cols, pruned_partition_list},- ListingTableUrl, PartitionedFile,-};-use crate::{- datasource::file_format::{file_compression_type::FileCompressionType, FileFormat},- datasource::physical_plan::FileSinkConfig,- execution::context::SessionState,-};-use arrow::datatypes::{DataType, Field, SchemaBuilder, SchemaRef};-use arrow_schema::Schema;+use crate::execution::SessionState;use async_trait::async_trait;-use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};-use datafusion_common::{- config_datafusion_err, config_err, internal_datafusion_err, internal_err, plan_err,- project_schema, stats::Precision, Constraints, DataFusionError, Result, SchemaExt,-};-use datafusion_datasource::{- compute_all_files_statistics,- file::FileSource,- file_groups::FileGroup,- file_scan_config::{FileScanConfig, FileScanConfigBuilder},- schema_adapter::{DefaultSchemaAdapterFactory, SchemaAdapter, SchemaAdapterFactory},-};-use datafusion_execution::{- cache::{cache_manager::FileStatisticsCache, cache_unit::DefaultFileStatisticsCache},- config::SessionConfig,-};-use datafusion_expr::execution_props::ExecutionProps;-use datafusion_expr::{- dml::InsertOp, Expr, SortExpr, TableProviderFilterPushDown, TableType,-};-use datafusion_physical_expr::create_lex_ordering;-use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;-use datafusion_physical_expr_common::sort_expr::LexOrdering;-use datafusion_physical_plan::{empty::EmptyExec, ExecutionPlan, Statistics};-use futures::{future, stream, Stream, StreamExt, TryStreamExt};-use itertools::Itertools;-use object_store::ObjectStore;-use std::{any::Any, collections::HashMap, str::FromStr, sync::Arc};--/// Indicates the source of the schema for a [`ListingTable`]-// PartialEq required for assert_eq! in tests-#[derive(Debug, Clone, Copy, PartialEq, Default)]-pub enum SchemaSource {- /// Schema is not yet set (initial state)- #[default]- Unset,- /// Schema was inferred from first table_path- Inferred,- /// Schema was specified explicitly via with_schema- Specified,-}+use datafusion_catalog_listing::{ListingOptions, ListingTableConfig};+use datafusion_common::{config_datafusion_err, internal_datafusion_err};+use datafusion_session::Session;+use futures::StreamExt;+use std::collections::HashMap;-/// Configuration for creating a [`ListingTable`]-///-/// # Schema Evolution Support-///-/// This configuration supports schema evolution through the optional-/// [`SchemaAdapterFactory`]. You might want to override the default factory when you need:+/// Extension trait for [`ListingTableConfig`] that supports inferring schemas///-/// - **Type coercion requirements**: When you need custom logic for converting between-/// different Arrow data types (e.g., Int32 ↔ Int64, Utf8 ↔ LargeUtf8)-/// - **Column mapping**: You need to map columns with a legacy name to a new name-/// - **Custom handling of missing columns**: By default they are filled in with nulls, but you may e.g. want to fill them in with `0` or `""`.-///-/// If not specified, a [`DefaultSchemaAdapterFactory`] will be used, which handles-/// basic schema compatibility cases.-///-#[derive(Debug, Clone, Default)]-pub struct ListingTableConfig {- /// Paths on the `ObjectStore` for creating `ListingTable`.- /// They should share the same schema and object store.- pub table_paths: Vec<ListingTableUrl>,- /// Optional `SchemaRef` for the to be created `ListingTable`.- ///- /// See details on [`ListingTableConfig::with_schema`]- pub file_schema: Option<SchemaRef>,- /// Optional [`ListingOptions`] for the to be created [`ListingTable`].- ///- /// See details on [`ListingTableConfig::with_listing_options`]- pub options: Option<ListingOptions>,- /// Tracks the source of the schema information- schema_source: SchemaSource,- /// Optional [`SchemaAdapterFactory`] for creating schema adapters- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,- /// Optional [`PhysicalExprAdapterFactory`] for creating physical expression adapters- expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,-}--impl ListingTableConfig {- /// Creates new [`ListingTableConfig`] for reading the specified URL- pub fn new(table_path: ListingTableUrl) -> Self {- Self {- table_paths: vec![table_path],- ..Default::default()- }- }-- /// Creates new [`ListingTableConfig`] with multiple table paths.- ///- /// See [`Self::infer_options`] for details on what happens with multiple paths- pub fn new_with_multi_paths(table_paths: Vec<ListingTableUrl>) -> Self {- Self {- table_paths,- ..Default::default()- }- }-- /// Returns the source of the schema for this configuration- pub fn schema_source(&self) -> SchemaSource {- self.schema_source- }- /// Set the `schema` for the overall [`ListingTable`]- ///- /// [`ListingTable`] will automatically coerce, when possible, the schema- /// for individual files to match this schema.- ///- /// If a schema is not provided, it is inferred using- /// [`Self::infer_schema`].- ///- /// If the schema is provided, it must contain only the fields in the file- /// without the table partitioning columns.- ///- /// # Example: Specifying Table Schema- /// ```rust- /// # use std::sync::Arc;- /// # use datafusion::datasource::listing::{ListingTableConfig, ListingOptions, ListingTableUrl};- /// # use datafusion::datasource::file_format::parquet::ParquetFormat;- /// # use arrow::datatypes::{Schema, Field, DataType};- /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap();- /// # let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default()));- /// let schema = Arc::new(Schema::new(vec![- /// Field::new("id", DataType::Int64, false),- /// Field::new("name", DataType::Utf8, true),- /// ]));- ///- /// let config = ListingTableConfig::new(table_paths)- /// .with_listing_options(listing_options) // Set options first- /// .with_schema(schema); // Then set schema- /// ```- pub fn with_schema(self, schema: SchemaRef) -> Self {- // Note: We preserve existing options state, but downstream code may expect- // options to be set. Consider calling with_listing_options() or infer_options()- // before operations that require options to be present.- debug_assert!(- self.options.is_some() || cfg!(test),- "ListingTableConfig::with_schema called without options set. \- Consider calling with_listing_options() or infer_options() first to avoid panics in downstream code."- );-- Self {- file_schema: Some(schema),- schema_source: SchemaSource::Specified,- ..self- }- }-- /// Add `listing_options` to [`ListingTableConfig`]- ///- /// If not provided, format and other options are inferred via- /// [`Self::infer_options`].- ///- /// # Example: Configuring Parquet Files with Custom Options- /// ```rust- /// # use std::sync::Arc;- /// # use datafusion::datasource::listing::{ListingTableConfig, ListingOptions, ListingTableUrl};- /// # use datafusion::datasource::file_format::parquet::ParquetFormat;- /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap();- /// let options = ListingOptions::new(Arc::new(ParquetFormat::default()))- /// .with_file_extension(".parquet")- /// .with_collect_stat(true);- ///- /// let config = ListingTableConfig::new(table_paths)- /// .with_listing_options(options); // Configure file format and options- /// ```- pub fn with_listing_options(self, listing_options: ListingOptions) -> Self {- // Note: This method properly sets options, but be aware that downstream- // methods like infer_schema() and try_new() require both schema and options- // to be set to function correctly.- debug_assert!(- !self.table_paths.is_empty() || cfg!(test),- "ListingTableConfig::with_listing_options called without table_paths set. \- Consider calling new() or new_with_multi_paths() first to establish table paths."- );-- Self {- options: Some(listing_options),- ..self- }- }-- /// Returns a tuple of `(file_extension, optional compression_extension)`- ///- /// For example a path ending with blah.test.csv.gz returns `("csv", Some("gz"))`- /// For example a path ending with blah.test.csv returns `("csv", None)`- fn infer_file_extension_and_compression_type(- path: &str,- ) -> Result<(String, Option<String>)> {- let mut exts = path.rsplit('.');-- let split = exts.next().unwrap_or("");-- let file_compression_type = FileCompressionType::from_str(split)- .unwrap_or(FileCompressionType::UNCOMPRESSED);-- if file_compression_type.is_compressed() {- let split2 = exts.next().unwrap_or("");- Ok((split2.to_string(), Some(split.to_string())))- } else {- Ok((split.to_string(), None))- }- }-+/// This trait exists because the following inference methods only+/// work for [`SessionState`] implementations of [`Session`].+/// See [`ListingTableConfig`] for the remaining inference methods.+#[async_trait]+pub trait ListingTableConfigExt {/// Infer `ListingOptions` based on `table_path` and file suffix.////// The format is inferred based on the first `table_path`.- pub async fn infer_options(self, state: &dyn Session) -> Result<Self> {+ async fn infer_options(+ self,+ state: &dyn Session,+ ) -> datafusion_common::Result<ListingTableConfig>;++ /// Convenience method to call both [`Self::infer_options`] and [`ListingTableConfig::infer_schema`]+ async fn infer(+ self,+ state: &dyn Session,+ ) -> datafusion_common::Result<ListingTableConfig>;+}++#[async_trait]+impl ListingTableConfigExt for ListingTableConfig {+ async fn infer_options(+ self,+ state: &dyn Session,+ ) -> datafusion_common::Result<ListingTableConfig> {let store = if let Some(url) = self.table_paths.first() {state.runtime_env().object_store(url)?} else {@@ -281,1299 +97,19 @@ impl ListingTableConfig {.with_target_partitions(state.config().target_partitions()).with_collect_stat(state.config().collect_statistics());- Ok(Self {- table_paths: self.table_paths,- file_schema: self.file_schema,- options: Some(listing_options),- schema_source: self.schema_source,- schema_adapter_factory: self.schema_adapter_factory,- expr_adapter_factory: self.expr_adapter_factory,- })- }-- /// Infer the [`SchemaRef`] based on `table_path`s.- ///- /// This method infers the table schema using the first `table_path`.- /// See [`ListingOptions::infer_schema`] for more details- ///- /// # Errors- /// * if `self.options` is not set. See [`Self::with_listing_options`]- pub async fn infer_schema(self, state: &dyn Session) -> Result<Self> {- match self.options {- Some(options) => {- let ListingTableConfig {- table_paths,- file_schema,- options: _,- schema_source,- schema_adapter_factory,- expr_adapter_factory: physical_expr_adapter_factory,- } = self;-- let (schema, new_schema_source) = match file_schema {- Some(schema) => (schema, schema_source), // Keep existing source if schema exists- None => {- if let Some(url) = table_paths.first() {- (- options.infer_schema(state, url).await?,- SchemaSource::Inferred,- )- } else {- (Arc::new(Schema::empty()), SchemaSource::Inferred)- }- }- };-- Ok(Self {- table_paths,- file_schema: Some(schema),- options: Some(options),- schema_source: new_schema_source,- schema_adapter_factory,- expr_adapter_factory: physical_expr_adapter_factory,- })- }- None => internal_err!("No `ListingOptions` set for inferring schema"),- }+ Ok(self.with_listing_options(listing_options))}- /// Convenience method to call both [`Self::infer_options`] and [`Self::infer_schema`]- pub async fn infer(self, state: &dyn Session) -> Result<Self> {+ async fn infer(self, state: &dyn Session) -> datafusion_common::Result<Self> {self.infer_options(state).await?.infer_schema(state).await}-- /// Infer the partition columns from `table_paths`.- ///- /// # Errors- /// * if `self.options` is not set. See [`Self::with_listing_options`]- pub async fn infer_partitions_from_path(self, state: &dyn Session) -> Result<Self> {- match self.options {- Some(options) => {- let Some(url) = self.table_paths.first() else {- return config_err!("No table path found");- };- let partitions = options- .infer_partitions(state, url)- .await?- .into_iter()- .map(|col_name| {- (- col_name,- DataType::Dictionary(- Box::new(DataType::UInt16),- Box::new(DataType::Utf8),- ),- )- })- .collect::<Vec<_>>();- let options = options.with_table_partition_cols(partitions);- Ok(Self {- table_paths: self.table_paths,- file_schema: self.file_schema,- options: Some(options),- schema_source: self.schema_source,- schema_adapter_factory: self.schema_adapter_factory,- expr_adapter_factory: self.expr_adapter_factory,- })- }- None => config_err!("No `ListingOptions` set for inferring schema"),- }- }-- /// Set the [`SchemaAdapterFactory`] for the [`ListingTable`]- ///- /// The schema adapter factory is used to create schema adapters that can- /// handle schema evolution and type conversions when reading files with- /// different schemas than the table schema.- ///- /// If not provided, a default schema adapter factory will be used.- ///- /// # Example: Custom Schema Adapter for Type Coercion- /// ```rust- /// # use std::sync::Arc;- /// # use datafusion::datasource::listing::{ListingTableConfig, ListingOptions, ListingTableUrl};- /// # use datafusion::datasource::schema_adapter::{SchemaAdapterFactory, SchemaAdapter};- /// # use datafusion::datasource::file_format::parquet::ParquetFormat;- /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType};- /// #- /// # #[derive(Debug)]- /// # struct MySchemaAdapterFactory;- /// # impl SchemaAdapterFactory for MySchemaAdapterFactory {- /// # fn create(&self, _projected_table_schema: SchemaRef, _file_schema: SchemaRef) -> Box<dyn SchemaAdapter> {- /// # unimplemented!()- /// # }- /// # }- /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap();- /// # let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default()));- /// # let table_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));- /// let config = ListingTableConfig::new(table_paths)- /// .with_listing_options(listing_options)- /// .with_schema(table_schema)- /// .with_schema_adapter_factory(Arc::new(MySchemaAdapterFactory));- /// ```- pub fn with_schema_adapter_factory(- self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Self {- Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self- }- }-- /// Get the [`SchemaAdapterFactory`] for this configuration- pub fn schema_adapter_factory(&self) -> Option<&Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.as_ref()- }-- /// Set the [`PhysicalExprAdapterFactory`] for the [`ListingTable`]- ///- /// The expression adapter factory is used to create physical expression adapters that can- /// handle schema evolution and type conversions when evaluating expressions- /// with different schemas than the table schema.- ///- /// If not provided, a default physical expression adapter factory will be used unless a custom- /// `SchemaAdapterFactory` is set, in which case only the `SchemaAdapterFactory` will be used.- ///- /// See <https://github.com/apache/datafusion/issues/16800> for details on this transition.- pub fn with_expr_adapter_factory(- self,- expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,- ) -> Self {- Self {- expr_adapter_factory: Some(expr_adapter_factory),- ..self- }- }-}--/// Options for creating a [`ListingTable`]-#[derive(Clone, Debug)]-pub struct ListingOptions {- /// A suffix on which files should be filtered (leave empty to- /// keep all files on the path)- pub file_extension: String,- /// The file format- pub format: Arc<dyn FileFormat>,- /// The expected partition column names in the folder structure.- /// See [Self::with_table_partition_cols] for details- pub table_partition_cols: Vec<(String, DataType)>,- /// Set true to try to guess statistics from the files.- /// This can add a lot of overhead as it will usually require files- /// to be opened and at least partially parsed.- pub collect_stat: bool,- /// Group files to avoid that the number of partitions exceeds- /// this limit- pub target_partitions: usize,- /// Optional pre-known sort order(s). Must be `SortExpr`s.- ///- /// DataFusion may take advantage of this ordering to omit sorts- /// or use more efficient algorithms. Currently sortedness must be- /// provided if it is known by some external mechanism, but may in- /// the future be automatically determined, for example using- /// parquet metadata.- ///- /// See <https://github.com/apache/datafusion/issues/4177>- ///- /// NOTE: This attribute stores all equivalent orderings (the outer `Vec`)- /// where each ordering consists of an individual lexicographic- /// ordering (encapsulated by a `Vec<Expr>`). If there aren't- /// multiple equivalent orderings, the outer `Vec` will have a- /// single element.- pub file_sort_order: Vec<Vec<SortExpr>>,-}--impl ListingOptions {- /// Creates an options instance with the given format- /// Default values:- /// - use default file extension filter- /// - no input partition to discover- /// - one target partition- /// - do not collect statistics- pub fn new(format: Arc<dyn FileFormat>) -> Self {- Self {- file_extension: format.get_ext(),- format,- table_partition_cols: vec![],- collect_stat: false,- target_partitions: 1,- file_sort_order: vec![],- }- }-- /// Set options from [`SessionConfig`] and returns self.- ///- /// Currently this sets `target_partitions` and `collect_stat`- /// but if more options are added in the future that need to be coordinated- /// they will be synchronized through this method.- pub fn with_session_config_options(mut self, config: &SessionConfig) -> Self {- self = self.with_target_partitions(config.target_partitions());- self = self.with_collect_stat(config.collect_statistics());- self- }-- /// Set file extension on [`ListingOptions`] and returns self.- ///- /// # Example- /// ```- /// # use std::sync::Arc;- /// # use datafusion::prelude::SessionContext;- /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat};- ///- /// let listing_options = ListingOptions::new(Arc::new(- /// ParquetFormat::default()- /// ))- /// .with_file_extension(".parquet");- ///- /// assert_eq!(listing_options.file_extension, ".parquet");- /// ```- pub fn with_file_extension(mut self, file_extension: impl Into<String>) -> Self {- self.file_extension = file_extension.into();- self- }-- /// Optionally set file extension on [`ListingOptions`] and returns self.- ///- /// If `file_extension` is `None`, the file extension will not be changed- ///- /// # Example- /// ```- /// # use std::sync::Arc;- /// # use datafusion::prelude::SessionContext;- /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat};- /// let extension = Some(".parquet");- /// let listing_options = ListingOptions::new(Arc::new(- /// ParquetFormat::default()- /// ))- /// .with_file_extension_opt(extension);- ///- /// assert_eq!(listing_options.file_extension, ".parquet");- /// ```- pub fn with_file_extension_opt<S>(mut self, file_extension: Option<S>) -> Self- where- S: Into<String>,- {- if let Some(file_extension) = file_extension {- self.file_extension = file_extension.into();- }- self- }-- /// Set `table partition columns` on [`ListingOptions`] and returns self.- ///- /// "partition columns," used to support [Hive Partitioning], are- /// columns added to the data that is read, based on the folder- /// structure where the data resides.- ///- /// For example, give the following files in your filesystem:- ///- /// ```text- /// /mnt/nyctaxi/year=2022/month=01/tripdata.parquet- /// /mnt/nyctaxi/year=2021/month=12/tripdata.parquet- /// /mnt/nyctaxi/year=2021/month=11/tripdata.parquet- /// ```- ///- /// A [`ListingTable`] created at `/mnt/nyctaxi/` with partition- /// columns "year" and "month" will include new `year` and `month`- /// columns while reading the files. The `year` column would have- /// value `2022` and the `month` column would have value `01` for- /// the rows read from- /// `/mnt/nyctaxi/year=2022/month=01/tripdata.parquet`- ///- ///# Notes- ///- /// - If only one level (e.g. `year` in the example above) is- /// specified, the other levels are ignored but the files are- /// still read.- ///- /// - Files that don't follow this partitioning scheme will be- /// ignored.- ///- /// - Since the columns have the same value for all rows read from- /// each individual file (such as dates), they are typically- /// dictionary encoded for efficiency. You may use- /// [`wrap_partition_type_in_dict`] to request a- /// dictionary-encoded type.- ///- /// - The partition columns are solely extracted from the file path. Especially they are NOT part of the parquet files itself.- ///- /// # Example- ///- /// ```- /// # use std::sync::Arc;- /// # use arrow::datatypes::DataType;- /// # use datafusion::prelude::col;- /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat};- ///- /// // listing options for files with paths such as `/mnt/data/col_a=x/col_b=y/data.parquet`- /// // `col_a` and `col_b` will be included in the data read from those files- /// let listing_options = ListingOptions::new(Arc::new(- /// ParquetFormat::default()- /// ))- /// .with_table_partition_cols(vec![("col_a".to_string(), DataType::Utf8),- /// ("col_b".to_string(), DataType::Utf8)]);- ///- /// assert_eq!(listing_options.table_partition_cols, vec![("col_a".to_string(), DataType::Utf8),- /// ("col_b".to_string(), DataType::Utf8)]);- /// ```- ///- /// [Hive Partitioning]: https://docs.cloudera.com/HDPDocuments/HDP2/HDP-2.1.3/bk_system-admin-guide/content/hive_partitioned_tables.html- /// [`wrap_partition_type_in_dict`]: crate::datasource::physical_plan::wrap_partition_type_in_dict- pub fn with_table_partition_cols(- mut self,- table_partition_cols: Vec<(String, DataType)>,- ) -> Self {- self.table_partition_cols = table_partition_cols;- self- }-- /// Set stat collection on [`ListingOptions`] and returns self.- ///- /// ```- /// # use std::sync::Arc;- /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat};- ///- /// let listing_options = ListingOptions::new(Arc::new(- /// ParquetFormat::default()- /// ))- /// .with_collect_stat(true);- ///- /// assert_eq!(listing_options.collect_stat, true);- /// ```- pub fn with_collect_stat(mut self, collect_stat: bool) -> Self {- self.collect_stat = collect_stat;- self- }-- /// Set number of target partitions on [`ListingOptions`] and returns self.- ///- /// ```- /// # use std::sync::Arc;- /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat};- ///- /// let listing_options = ListingOptions::new(Arc::new(- /// ParquetFormat::default()- /// ))- /// .with_target_partitions(8);- ///- /// assert_eq!(listing_options.target_partitions, 8);- /// ```- pub fn with_target_partitions(mut self, target_partitions: usize) -> Self {- self.target_partitions = target_partitions;- self- }-- /// Set file sort order on [`ListingOptions`] and returns self.- ///- /// ```- /// # use std::sync::Arc;- /// # use datafusion::prelude::col;- /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat};- ///- /// // Tell datafusion that the files are sorted by column "a"- /// let file_sort_order = vec![vec![- /// col("a").sort(true, true)- /// ]];- ///- /// let listing_options = ListingOptions::new(Arc::new(- /// ParquetFormat::default()- /// ))- /// .with_file_sort_order(file_sort_order.clone());- ///- /// assert_eq!(listing_options.file_sort_order, file_sort_order);- /// ```- pub fn with_file_sort_order(mut self, file_sort_order: Vec<Vec<SortExpr>>) -> Self {- self.file_sort_order = file_sort_order;- self- }-- /// Infer the schema of the files at the given path on the provided object store.- ///- /// If the table_path contains one or more files (i.e. it is a directory /- /// prefix of files) their schema is merged by calling [`FileFormat::infer_schema`]- ///- /// Note: The inferred schema does not include any partitioning columns.- ///- /// This method is called as part of creating a [`ListingTable`].- pub async fn infer_schema<'a>(- &'a self,- state: &dyn Session,- table_path: &'a ListingTableUrl,- ) -> Result<SchemaRef> {- let store = state.runtime_env().object_store(table_path)?;-- let files: Vec<_> = table_path- .list_all_files(state, store.as_ref(), &self.file_extension)- .await?- // Empty files cannot affect schema but may throw when trying to read for it- .try_filter(|object_meta| future::ready(object_meta.size > 0))- .try_collect()- .await?;-- let schema = self.format.infer_schema(state, &store, &files).await?;-- Ok(schema)- }-- /// Infers the partition columns stored in `LOCATION` and compares- /// them with the columns provided in `PARTITIONED BY` to help prevent- /// accidental corrupts of partitioned tables.- ///- /// Allows specifying partial partitions.- pub async fn validate_partitions(- &self,- state: &dyn Session,- table_path: &ListingTableUrl,- ) -> Result<()> {- if self.table_partition_cols.is_empty() {- return Ok(());- }-- if !table_path.is_collection() {- return plan_err!(- "Can't create a partitioned table backed by a single file, \- perhaps the URL is missing a trailing slash?"- );- }-- let inferred = self.infer_partitions(state, table_path).await?;-- // no partitioned files found on disk- if inferred.is_empty() {- return Ok(());- }-- let table_partition_names = self- .table_partition_cols- .iter()- .map(|(col_name, _)| col_name.clone())- .collect_vec();-- if inferred.len() < table_partition_names.len() {- return plan_err!(- "Inferred partitions to be {:?}, but got {:?}",- inferred,- table_partition_names- );- }-- // match prefix to allow creating tables with partial partitions- for (idx, col) in table_partition_names.iter().enumerate() {- if &inferred[idx] != col {- return plan_err!(- "Inferred partitions to be {:?}, but got {:?}",- inferred,- table_partition_names- );- }- }-- Ok(())- }-- /// Infer the partitioning at the given path on the provided object store.- /// For performance reasons, it doesn't read all the files on disk- /// and therefore may fail to detect invalid partitioning.- pub(crate) async fn infer_partitions(- &self,- state: &dyn Session,- table_path: &ListingTableUrl,- ) -> Result<Vec<String>> {- let store = state.runtime_env().object_store(table_path)?;-- // only use 10 files for inference- // This can fail to detect inconsistent partition keys- // A DFS traversal approach of the store can help here- let files: Vec<_> = table_path- .list_all_files(state, store.as_ref(), &self.file_extension)- .await?- .take(10)- .try_collect()- .await?;-- let stripped_path_parts = files.iter().map(|file| {- table_path- .strip_prefix(&file.location)- .unwrap()- .collect_vec()- });-- let partition_keys = stripped_path_parts- .map(|path_parts| {- path_parts- .into_iter()- .rev()- .skip(1) // get parents only; skip the file itself- .rev()- // Partitions are expected to follow the format "column_name=value", so we- // should ignore any path part that cannot be parsed into the expected format- .filter(|s| s.contains('='))- .map(|s| s.split('=').take(1).collect())- .collect_vec()- })- .collect_vec();-- match partition_keys.into_iter().all_equal_value() {- Ok(v) => Ok(v),- Err(None) => Ok(vec![]),- Err(Some(diff)) => {- let mut sorted_diff = [diff.0, diff.1];- sorted_diff.sort();- plan_err!("Found mixed partition values on disk {:?}", sorted_diff)- }- }- }-}--/// Built in [`TableProvider`] that reads data from one or more files as a single table.-///-/// The files are read using an [`ObjectStore`] instance, for example from-/// local files or objects from AWS S3.-///-/// # Features:-/// * Reading multiple files as a single table-/// * Hive style partitioning (e.g., directories named `date=2024-06-01`)-/// * Merges schemas from files with compatible but not identical schemas (see [`ListingTableConfig::file_schema`])-/// * `limit`, `filter` and `projection` pushdown for formats that support it (e.g.,-/// Parquet)-/// * Statistics collection and pruning based on file metadata-/// * Pre-existing sort order (see [`ListingOptions::file_sort_order`])-/// * Metadata caching to speed up repeated queries (see [`FileMetadataCache`])-/// * Statistics caching (see [`FileStatisticsCache`])-///-/// [`FileMetadataCache`]: datafusion_execution::cache::cache_manager::FileMetadataCache-///-/// # Reading Directories and Hive Style Partitioning-///-/// For example, given the `table1` directory (or object store prefix)-///-/// ```text-/// table1-/// ├── file1.parquet-/// └── file2.parquet-/// ```-///-/// A `ListingTable` would read the files `file1.parquet` and `file2.parquet` as-/// a single table, merging the schemas if the files have compatible but not-/// identical schemas.-///-/// Given the `table2` directory (or object store prefix)-///-/// ```text-/// table2-/// ├── date=2024-06-01-/// │ ├── file3.parquet-/// │ └── file4.parquet-/// └── date=2024-06-02-/// └── file5.parquet-/// ```-///-/// A `ListingTable` would read the files `file3.parquet`, `file4.parquet`, and-/// `file5.parquet` as a single table, again merging schemas if necessary.-///-/// Given the hive style partitioning structure (e.g,. directories named-/// `date=2024-06-01` and `date=2026-06-02`), `ListingTable` also adds a `date`-/// column when reading the table:-/// * The files in `table2/date=2024-06-01` will have the value `2024-06-01`-/// * The files in `table2/date=2024-06-02` will have the value `2024-06-02`.-///-/// If the query has a predicate like `WHERE date = '2024-06-01'`-/// only the corresponding directory will be read.-///-/// # See Also-///-/// 1. [`ListingTableConfig`]: Configuration options-/// 1. [`DataSourceExec`]: `ExecutionPlan` used by `ListingTable`-///-/// [`DataSourceExec`]: crate::datasource::source::DataSourceExec-///-/// # Caching Metadata-///-/// Some formats, such as Parquet, use the `FileMetadataCache` to cache file-/// metadata that is needed to execute but expensive to read, such as row-/// groups and statistics. The cache is scoped to the [`SessionContext`] and can-/// be configured via the [runtime config options].-///-/// [`SessionContext`]: crate::prelude::SessionContext-/// [runtime config options]: https://datafusion.apache.org/user-guide/configs.html#runtime-configuration-settings-///-/// # Example: Read a directory of parquet files using a [`ListingTable`]-///-/// ```no_run-/// # use datafusion::prelude::SessionContext;-/// # use datafusion::error::Result;-/// # use std::sync::Arc;-/// # use datafusion::datasource::{-/// # listing::{-/// # ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,-/// # },-/// # file_format::parquet::ParquetFormat,-/// # };-/// # #[tokio::main]-/// # async fn main() -> Result<()> {-/// let ctx = SessionContext::new();-/// let session_state = ctx.state();-/// let table_path = "/path/to/parquet";-///-/// // Parse the path-/// let table_path = ListingTableUrl::parse(table_path)?;-///-/// // Create default parquet options-/// let file_format = ParquetFormat::new();-/// let listing_options = ListingOptions::new(Arc::new(file_format))-/// .with_file_extension(".parquet");-///-/// // Resolve the schema-/// let resolved_schema = listing_options-/// .infer_schema(&session_state, &table_path)-/// .await?;-///-/// let config = ListingTableConfig::new(table_path)-/// .with_listing_options(listing_options)-/// .with_schema(resolved_schema);-///-/// // Create a new TableProvider-/// let provider = Arc::new(ListingTable::try_new(config)?);-///-/// // This provider can now be read as a dataframe:-/// let df = ctx.read_table(provider.clone());-///-/// // or registered as a named table:-/// ctx.register_table("my_table", provider);-///-/// # Ok(())-/// # }-/// ```-#[derive(Debug, Clone)]-pub struct ListingTable {- table_paths: Vec<ListingTableUrl>,- /// `file_schema` contains only the columns physically stored in the data files themselves.- /// - Represents the actual fields found in files like Parquet, CSV, etc.- /// - Used when reading the raw data from files- file_schema: SchemaRef,- /// `table_schema` combines `file_schema` + partition columns- /// - Partition columns are derived from directory paths (not stored in files)- /// - These are columns like "year=2022/month=01" in paths like `/data/year=2022/month=01/file.parquet`- table_schema: SchemaRef,- /// Indicates how the schema was derived (inferred or explicitly specified)- schema_source: SchemaSource,- /// Options used to configure the listing table such as the file format- /// and partitioning information- options: ListingOptions,- /// The SQL definition for this table, if any- definition: Option<String>,- /// Cache for collected file statistics- collected_statistics: FileStatisticsCache,- /// Constraints applied to this table- constraints: Constraints,- /// Column default expressions for columns that are not physically present in the data files- column_defaults: HashMap<String, Expr>,- /// Optional [`SchemaAdapterFactory`] for creating schema adapters- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,- /// Optional [`PhysicalExprAdapterFactory`] for creating physical expression adapters- expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,-}--impl ListingTable {- /// Create new [`ListingTable`]- ///- /// See documentation and example on [`ListingTable`] and [`ListingTableConfig`]- pub fn try_new(config: ListingTableConfig) -> Result<Self> {- // Extract schema_source before moving other parts of the config- let schema_source = config.schema_source();-- let file_schema = config- .file_schema- .ok_or_else(|| internal_datafusion_err!("No schema provided."))?;-- let options = config- .options- .ok_or_else(|| internal_datafusion_err!("No ListingOptions provided"))?;-- // Add the partition columns to the file schema- let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned());- for (part_col_name, part_col_type) in &options.table_partition_cols {- builder.push(Field::new(part_col_name, part_col_type.clone(), false));- }-- let table_schema = Arc::new(- builder- .finish()- .with_metadata(file_schema.metadata().clone()),- );-- let table = Self {- table_paths: config.table_paths,- file_schema,- table_schema,- schema_source,- options,- definition: None,- collected_statistics: Arc::new(DefaultFileStatisticsCache::default()),- constraints: Constraints::default(),- column_defaults: HashMap::new(),- schema_adapter_factory: config.schema_adapter_factory,- expr_adapter_factory: config.expr_adapter_factory,- };-- Ok(table)- }-- /// Assign constraints- pub fn with_constraints(mut self, constraints: Constraints) -> Self {- self.constraints = constraints;- self- }-- /// Assign column defaults- pub fn with_column_defaults(- mut self,- column_defaults: HashMap<String, Expr>,- ) -> Self {- self.column_defaults = column_defaults;- self- }-- /// Set the [`FileStatisticsCache`] used to cache parquet file statistics.- ///- /// Setting a statistics cache on the `SessionContext` can avoid refetching statistics- /// multiple times in the same session.- ///- /// If `None`, creates a new [`DefaultFileStatisticsCache`] scoped to this query.- pub fn with_cache(mut self, cache: Option<FileStatisticsCache>) -> Self {- self.collected_statistics =- cache.unwrap_or_else(|| Arc::new(DefaultFileStatisticsCache::default()));- self- }-- /// Specify the SQL definition for this table, if any- pub fn with_definition(mut self, definition: Option<String>) -> Self {- self.definition = definition;- self- }-- /// Get paths ref- pub fn table_paths(&self) -> &Vec<ListingTableUrl> {- &self.table_paths- }-- /// Get options ref- pub fn options(&self) -> &ListingOptions {- &self.options- }-- /// Get the schema source- pub fn schema_source(&self) -> SchemaSource {- self.schema_source- }-- /// Set the [`SchemaAdapterFactory`] for this [`ListingTable`]- ///- /// The schema adapter factory is used to create schema adapters that can- /// handle schema evolution and type conversions when reading files with- /// different schemas than the table schema.- ///- /// # Example: Adding Schema Evolution Support- /// ```rust- /// # use std::sync::Arc;- /// # use datafusion::datasource::listing::{ListingTable, ListingTableConfig, ListingOptions, ListingTableUrl};- /// # use datafusion::datasource::schema_adapter::{DefaultSchemaAdapterFactory, SchemaAdapter};- /// # use datafusion::datasource::file_format::parquet::ParquetFormat;- /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType};- /// # let table_path = ListingTableUrl::parse("file:///path/to/data").unwrap();- /// # let options = ListingOptions::new(Arc::new(ParquetFormat::default()));- /// # let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));- /// # let config = ListingTableConfig::new(table_path).with_listing_options(options).with_schema(schema);- /// # let table = ListingTable::try_new(config).unwrap();- /// let table_with_evolution = table- /// .with_schema_adapter_factory(Arc::new(DefaultSchemaAdapterFactory));- /// ```- /// See [`ListingTableConfig::with_schema_adapter_factory`] for an example of custom SchemaAdapterFactory.- pub fn with_schema_adapter_factory(- self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Self {- Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self- }- }-- /// Get the [`SchemaAdapterFactory`] for this table- pub fn schema_adapter_factory(&self) -> Option<&Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.as_ref()- }-- /// Creates a schema adapter for mapping between file and table schemas- ///- /// Uses the configured schema adapter factory if available, otherwise falls back- /// to the default implementation.- fn create_schema_adapter(&self) -> Box<dyn SchemaAdapter> {- let table_schema = self.schema();- match &self.schema_adapter_factory {- Some(factory) => {- factory.create_with_projected_schema(Arc::clone(&table_schema))- }- None => DefaultSchemaAdapterFactory::from_schema(Arc::clone(&table_schema)),- }- }-- /// Creates a file source and applies schema adapter factory if available- fn create_file_source_with_schema_adapter(&self) -> Result<Arc<dyn FileSource>> {- let mut source = self.options.format.file_source();- // Apply schema adapter to source if available- //- // The source will use this SchemaAdapter to adapt data batches as they flow up the plan.- // Note: ListingTable also creates a SchemaAdapter in `scan()` but that is only used to adapt collected statistics.- if let Some(factory) = &self.schema_adapter_factory {- source = source.with_schema_adapter_factory(Arc::clone(factory))?;- }- Ok(source)- }-- /// If file_sort_order is specified, creates the appropriate physical expressions- fn try_create_output_ordering(- &self,- execution_props: &ExecutionProps,- ) -> Result<Vec<LexOrdering>> {- create_lex_ordering(- &self.table_schema,- &self.options.file_sort_order,- execution_props,- )- }-}--// Expressions can be used for partition pruning if they can be evaluated using-// only the partition columns and there are partition columns.-fn can_be_evaluated_for_partition_pruning(- partition_column_names: &[&str],- expr: &Expr,-) -> bool {- !partition_column_names.is_empty()- && expr_applicable_for_cols(partition_column_names, expr)-}--#[async_trait]-impl TableProvider for ListingTable {- fn as_any(&self) -> &dyn Any {- self- }-- fn schema(&self) -> SchemaRef {- Arc::clone(&self.table_schema)- }-- fn constraints(&self) -> Option<&Constraints> {- Some(&self.constraints)- }-- fn table_type(&self) -> TableType {- TableType::Base- }-- async fn scan(- &self,- state: &dyn Session,- projection: Option<&Vec<usize>>,- filters: &[Expr],- limit: Option<usize>,- ) -> Result<Arc<dyn ExecutionPlan>> {- let options = ScanArgs::default()- .with_projection(projection.map(|p| p.as_slice()))- .with_filters(Some(filters))- .with_limit(limit);- Ok(self.scan_with_args(state, options).await?.into_inner())- }-- async fn scan_with_args<'a>(- &self,- state: &dyn Session,- args: ScanArgs<'a>,- ) -> Result<ScanResult> {- let projection = args.projection().map(|p| p.to_vec());- let filters = args.filters().map(|f| f.to_vec()).unwrap_or_default();- let limit = args.limit();-- // extract types of partition columns- let table_partition_cols = self- .options- .table_partition_cols- .iter()- .map(|col| Ok(self.table_schema.field_with_name(&col.0)?.clone()))- .collect::<Result<Vec<_>>>()?;-- let table_partition_col_names = table_partition_cols- .iter()- .map(|field| field.name().as_str())- .collect::<Vec<_>>();-- // If the filters can be resolved using only partition cols, there is no need to- // pushdown it to TableScan, otherwise, `unhandled` pruning predicates will be generated- let (partition_filters, filters): (Vec<_>, Vec<_>) =- filters.iter().cloned().partition(|filter| {- can_be_evaluated_for_partition_pruning(&table_partition_col_names, filter)- });-- // We should not limit the number of partitioned files to scan if there are filters and limit- // at the same time. This is because the limit should be applied after the filters are applied.- let statistic_file_limit = if filters.is_empty() { limit } else { None };-- let (mut partitioned_file_lists, statistics) = self- .list_files_for_scan(state, &partition_filters, statistic_file_limit)- .await?;-- // if no files need to be read, return an `EmptyExec`- if partitioned_file_lists.is_empty() {- let projected_schema = project_schema(&self.schema(), projection.as_ref())?;- return Ok(ScanResult::new(Arc::new(EmptyExec::new(projected_schema))));- }-- let output_ordering = self.try_create_output_ordering(state.execution_props())?;- match state- .config_options()- .execution- .split_file_groups_by_statistics- .then(|| {- output_ordering.first().map(|output_ordering| {- FileScanConfig::split_groups_by_statistics_with_target_partitions(- &self.table_schema,- &partitioned_file_lists,- output_ordering,- self.options.target_partitions,- )- })- })- .flatten()- {- Some(Err(e)) => log::debug!("failed to split file groups by statistics: {e}"),- Some(Ok(new_groups)) => {- if new_groups.len() <= self.options.target_partitions {- partitioned_file_lists = new_groups;- } else {- log::debug!("attempted to split file groups by statistics, but there were more file groups than target_partitions; falling back to unordered")- }- }- None => {} // no ordering required- };-- let Some(object_store_url) =- self.table_paths.first().map(ListingTableUrl::object_store)- else {- return Ok(ScanResult::new(Arc::new(EmptyExec::new(Arc::new(- Schema::empty(),- )))));- };-- let file_source = self.create_file_source_with_schema_adapter()?;-- // create the execution plan- let plan = self- .options- .format- .create_physical_plan(- state,- FileScanConfigBuilder::new(- object_store_url,- Arc::clone(&self.file_schema),- file_source,- )- .with_file_groups(partitioned_file_lists)- .with_constraints(self.constraints.clone())- .with_statistics(statistics)- .with_projection(projection)- .with_limit(limit)- .with_output_ordering(output_ordering)- .with_table_partition_cols(table_partition_cols)- .with_expr_adapter(self.expr_adapter_factory.clone())- .build(),- )- .await?;-- Ok(ScanResult::new(plan))- }-- fn supports_filters_pushdown(- &self,- filters: &[&Expr],- ) -> Result<Vec<TableProviderFilterPushDown>> {- let partition_column_names = self- .options- .table_partition_cols- .iter()- .map(|col| col.0.as_str())- .collect::<Vec<_>>();- filters- .iter()- .map(|filter| {- if can_be_evaluated_for_partition_pruning(&partition_column_names, filter)- {- // if filter can be handled by partition pruning, it is exact- return Ok(TableProviderFilterPushDown::Exact);- }-- Ok(TableProviderFilterPushDown::Inexact)- })- .collect()- }-- fn get_table_definition(&self) -> Option<&str> {- self.definition.as_deref()- }-- async fn insert_into(- &self,- state: &dyn Session,- input: Arc<dyn ExecutionPlan>,- insert_op: InsertOp,- ) -> Result<Arc<dyn ExecutionPlan>> {- // Check that the schema of the plan matches the schema of this table.- self.schema()- .logically_equivalent_names_and_types(&input.schema())?;-- let table_path = &self.table_paths()[0];- if !table_path.is_collection() {- return plan_err!(- "Inserting into a ListingTable backed by a single file is not supported, URL is possibly missing a trailing `/`. \- To append to an existing file use StreamTable, e.g. by using CREATE UNBOUNDED EXTERNAL TABLE"- );- }-- // Get the object store for the table path.- let store = state.runtime_env().object_store(table_path)?;-- let file_list_stream = pruned_partition_list(- state,- store.as_ref(),- table_path,- &[],- &self.options.file_extension,- &self.options.table_partition_cols,- )- .await?;-- let file_group = file_list_stream.try_collect::<Vec<_>>().await?.into();- let keep_partition_by_columns =- state.config_options().execution.keep_partition_by_columns;-- // Sink related option, apart from format- let config = FileSinkConfig {- original_url: String::default(),- object_store_url: self.table_paths()[0].object_store(),- table_paths: self.table_paths().clone(),- file_group,- output_schema: self.schema(),- table_partition_cols: self.options.table_partition_cols.clone(),- insert_op,- keep_partition_by_columns,- file_extension: self.options().format.get_ext(),- };-- let orderings = self.try_create_output_ordering(state.execution_props())?;- // It is sufficient to pass only one of the equivalent orderings:- let order_requirements = orderings.into_iter().next().map(Into::into);-- self.options()- .format- .create_writer_physical_plan(input, state, config, order_requirements)- .await- }-- fn get_column_default(&self, column: &str) -> Option<&Expr> {- self.column_defaults.get(column)- }-}--impl ListingTable {- /// Get the list of files for a scan as well as the file level statistics.- /// The list is grouped to let the execution plan know how the files should- /// be distributed to different threads / executors.- async fn list_files_for_scan<'a>(- &'a self,- ctx: &'a dyn Session,- filters: &'a [Expr],- limit: Option<usize>,- ) -> Result<(Vec<FileGroup>, Statistics)> {- let store = if let Some(url) = self.table_paths.first() {- ctx.runtime_env().object_store(url)?- } else {- return Ok((vec![], Statistics::new_unknown(&self.file_schema)));- };- // list files (with partitions)- let file_list = future::try_join_all(self.table_paths.iter().map(|table_path| {- pruned_partition_list(- ctx,- store.as_ref(),- table_path,- filters,- &self.options.file_extension,- &self.options.table_partition_cols,- )- }))- .await?;- let meta_fetch_concurrency =- ctx.config_options().execution.meta_fetch_concurrency;- let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency);- // collect the statistics if required by the config- let files = file_list- .map(|part_file| async {- let part_file = part_file?;- let statistics = if self.options.collect_stat {- self.do_collect_statistics(ctx, &store, &part_file).await?- } else {- Arc::new(Statistics::new_unknown(&self.file_schema))- };- Ok(part_file.with_statistics(statistics))- })- .boxed()- .buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency);-- let (file_group, inexact_stats) =- get_files_with_limit(files, limit, self.options.collect_stat).await?;-- let file_groups = file_group.split_files(self.options.target_partitions);- let (mut file_groups, mut stats) = compute_all_files_statistics(- file_groups,- self.schema(),- self.options.collect_stat,- inexact_stats,- )?;-- let schema_adapter = self.create_schema_adapter();- let (schema_mapper, _) = schema_adapter.map_schema(self.file_schema.as_ref())?;-- stats.column_statistics =- schema_mapper.map_column_statistics(&stats.column_statistics)?;- file_groups.iter_mut().try_for_each(|file_group| {- if let Some(stat) = file_group.statistics_mut() {- stat.column_statistics =- schema_mapper.map_column_statistics(&stat.column_statistics)?;- }- Ok::<_, DataFusionError>(())- })?;- Ok((file_groups, stats))- }-- /// Collects statistics for a given partitioned file.- ///- /// This method first checks if the statistics for the given file are already cached.- /// If they are, it returns the cached statistics.- /// If they are not, it infers the statistics from the file and stores them in the cache.- async fn do_collect_statistics(- &self,- ctx: &dyn Session,- store: &Arc<dyn ObjectStore>,- part_file: &PartitionedFile,- ) -> Result<Arc<Statistics>> {- match self- .collected_statistics- .get_with_extra(&part_file.object_meta.location, &part_file.object_meta)- {- Some(statistics) => Ok(statistics),- None => {- let statistics = self- .options- .format- .infer_stats(- ctx,- store,- Arc::clone(&self.file_schema),- &part_file.object_meta,- )- .await?;- let statistics = Arc::new(statistics);- self.collected_statistics.put_with_extra(- &part_file.object_meta.location,- Arc::clone(&statistics),- &part_file.object_meta,- );- Ok(statistics)- }- }- }-}--/// Processes a stream of partitioned files and returns a `FileGroup` containing the files.-///-/// This function collects files from the provided stream until either:-/// 1. The stream is exhausted-/// 2. The accumulated number of rows exceeds the provided `limit` (if specified)-///-/// # Arguments-/// * `files` - A stream of `Result<PartitionedFile>` items to process-/// * `limit` - An optional row count limit. If provided, the function will stop collecting files-/// once the accumulated number of rows exceeds this limit-/// * `collect_stats` - Whether to collect and accumulate statistics from the files-///-/// # Returns-/// A `Result` containing a `FileGroup` with the collected files-/// and a boolean indicating whether the statistics are inexact.-///-/// # Note-/// The function will continue processing files if statistics are not available or if the-/// limit is not provided. If `collect_stats` is false, statistics won't be accumulated-/// but files will still be collected.-async fn get_files_with_limit(- files: impl Stream<Item = Result<PartitionedFile>>,- limit: Option<usize>,- collect_stats: bool,-) -> Result<(FileGroup, bool)> {- let mut file_group = FileGroup::default();- // Fusing the stream allows us to call next safely even once it is finished.- let mut all_files = Box::pin(files.fuse());- enum ProcessingState {- ReadingFiles,- ReachedLimit,- }-- let mut state = ProcessingState::ReadingFiles;- let mut num_rows = Precision::Absent;-- while let Some(file_result) = all_files.next().await {- // Early exit if we've already reached our limit- if matches!(state, ProcessingState::ReachedLimit) {- break;- }-- let file = file_result?;-- // Update file statistics regardless of state- if collect_stats {- if let Some(file_stats) = &file.statistics {- num_rows = if file_group.is_empty() {- // For the first file, just take its row count- file_stats.num_rows- } else {- // For subsequent files, accumulate the counts- num_rows.add(&file_stats.num_rows)- };- }- }-- // Always add the file to our group- file_group.push(file);-- // Check if we've hit the limit (if one was specified)- if let Some(limit) = limit {- if let Precision::Exact(row_count) = num_rows {- if row_count > limit {- state = ProcessingState::ReachedLimit;- }- }- }- }- // If we still have files in the stream, it means that the limit kicked- // in, and the statistic could have been different had we processed the- // files in a different order.- let inexact_stats = all_files.next().await.is_some();- Ok((file_group, inexact_stats))}#[cfg(test)]mod tests {- use super::*;#[cfg(feature = "parquet")]use crate::datasource::file_format::parquet::ParquetFormat;+ use crate::datasource::listing::table::ListingTableConfigExt;use crate::prelude::*;use crate::{datasource::{@@ -1587,21 +123,34 @@ mod tests {},};use arrow::{compute::SortOptions, record_batch::RecordBatch};+ use arrow_schema::{DataType, Field, Schema, SchemaRef};+ use datafusion_catalog::TableProvider;+ use datafusion_catalog_listing::{+ ListingOptions, ListingTable, ListingTableConfig, SchemaSource,+ };use datafusion_common::{- assert_contains,+ assert_contains, plan_err,stats::Precision,test_util::{batches_to_string, datafusion_test_data},- ColumnStatistics, ScalarValue,+ ColumnStatistics, DataFusionError, Result, ScalarValue,};+ use datafusion_datasource::file_compression_type::FileCompressionType;+ use datafusion_datasource::file_format::FileFormat;use datafusion_datasource::schema_adapter::{SchemaAdapter, SchemaAdapterFactory, SchemaMapper,};+ use datafusion_datasource::ListingTableUrl;+ use datafusion_expr::dml::InsertOp;use datafusion_expr::{BinaryExpr, LogicalPlanBuilder, Operator};use datafusion_physical_expr::expressions::binary;use datafusion_physical_expr::PhysicalSortExpr;+ use datafusion_physical_expr_common::sort_expr::LexOrdering;+ use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::{collect, ExecutionPlanProperties};use rstest::rstest;+ use std::collections::HashMap;use std::io::Write;+ use std::sync::Arc;use tempfile::TempDir;use url::Url;@@ -1638,10 +187,13 @@ mod tests {let ctx = SessionContext::new();let testdata = datafusion_test_data();let filename = format!("{testdata}/aggregate_simple.csv");- let table_path = ListingTableUrl::parse(filename).unwrap();+ let table_path = ListingTableUrl::parse(filename)?;// Test default schema source- let config = ListingTableConfig::new(table_path.clone());+ let format = CsvFormat::default();+ let options = ListingOptions::new(Arc::new(format));+ let config =+ ListingTableConfig::new(table_path.clone()).with_listing_options(options);assert_eq!(config.schema_source(), SchemaSource::Unset);// Test schema source after setting a schema explicitly@@ -1650,18 +202,13 @@ mod tests {assert_eq!(config_with_schema.schema_source(), SchemaSource::Specified);// Test schema source after inferring schema- let format = CsvFormat::default();- let options = ListingOptions::new(Arc::new(format));- let config_with_options = config.with_listing_options(options.clone());- assert_eq!(config_with_options.schema_source(), SchemaSource::Unset);+ assert_eq!(config.schema_source(), SchemaSource::Unset);- let config_with_inferred = config_with_options.infer_schema(&ctx.state()).await?;+ let config_with_inferred = config.infer_schema(&ctx.state()).await?;assert_eq!(config_with_inferred.schema_source(), SchemaSource::Inferred);// Test schema preservation through operations- let config_with_schema_and_options = config_with_schema- .clone()- .with_listing_options(options.clone());+ let config_with_schema_and_options = config_with_schema.clone();assert_eq!(config_with_schema_and_options.schema_source(),SchemaSource::Specified@@ -1836,7 +383,7 @@ mod tests {.with_table_partition_cols(vec![(String::from("p1"), DataType::Utf8)]).with_target_partitions(4);- let table_path = ListingTableUrl::parse("test:///table/").unwrap();+ let table_path = ListingTableUrl::parse("test:///table/")?;let file_schema =Arc::new(Schema::new(vec![Field::new("a", DataType::Boolean, false)]));let config = ListingTableConfig::new(table_path)@@ -1872,7 +419,7 @@ mod tests {) -> Result<Arc<dyn TableProvider>> {let testdata = crate::test_util::parquet_test_data();let filename = format!("{testdata}/{name}");- let table_path = ListingTableUrl::parse(filename).unwrap();+ let table_path = ListingTableUrl::parse(filename)?;let config = ListingTableConfig::new(table_path).infer(&ctx.state())@@ -1899,7 +446,7 @@ mod tests {let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]);- let table_path = ListingTableUrl::parse(table_prefix).unwrap();+ let table_path = ListingTableUrl::parse(table_prefix)?;let config = ListingTableConfig::new(table_path).with_listing_options(opt).with_schema(Arc::new(schema));@@ -2458,7 +1005,7 @@ mod tests {async fn test_infer_options_compressed_csv() -> Result<()> {let testdata = crate::test_util::arrow_test_data();let filename = format!("{testdata}/csv/aggregate_test_100.csv.gz");- let table_path = ListingTableUrl::parse(filename).unwrap();+ let table_path = ListingTableUrl::parse(filename)?;let ctx = SessionContext::new();@@ -2479,12 +1026,15 @@ mod tests {let testdata = datafusion_test_data();let filename = format!("{testdata}/aggregate_simple.csv");- let table_path = ListingTableUrl::parse(filename).unwrap();+ let table_path = ListingTableUrl::parse(filename)?;let provided_schema = create_test_schema();- let config =- ListingTableConfig::new(table_path).with_schema(Arc::clone(&provided_schema));+ let format = CsvFormat::default();+ let options = ListingOptions::new(Arc::new(format));+ let config = ListingTableConfig::new(table_path)+ .with_listing_options(options)+ .with_schema(Arc::clone(&provided_schema));let config = config.infer(&ctx.state()).await?;@@ -2549,8 +1099,8 @@ mod tests {table_path1.clone(),table_path2.clone(),])- .with_schema(schema_3cols)- .with_listing_options(options.clone());+ .with_listing_options(options.clone())+ .with_schema(schema_3cols);let config2 = config2.infer_schema(&ctx.state()).await?;assert_eq!(config2.schema_source(), SchemaSource::Specified);@@ -2573,8 +1123,8 @@ mod tests {table_path1.clone(),table_path2.clone(),])- .with_schema(schema_4cols)- .with_listing_options(options.clone());+ .with_listing_options(options.clone())+ .with_schema(schema_4cols);let config3 = config3.infer_schema(&ctx.state()).await?;assert_eq!(config3.schema_source(), SchemaSource::Specified);@@ -2785,7 +1335,7 @@ mod tests {let testdata = crate::test_util::parquet_test_data();let filename = format!("{}/{}", testdata, "alltypes_plain.parquet");- let table_path = ListingTableUrl::parse(filename).unwrap();+ let table_path = ListingTableUrl::parse(filename)?;let ctx = SessionContext::new();let state = ctx.state();@@ -2932,7 +1482,7 @@ mod tests {let format = JsonFormat::default();let opt = ListingOptions::new(Arc::new(format)).with_collect_stat(false);let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]);- let table_path = ListingTableUrl::parse("test:///table/").unwrap();+ let table_path = ListingTableUrl::parse("test:///table/")?;let config = ListingTableConfig::new(table_path).with_listing_options(opt)@@ -3146,7 +1696,7 @@ mod tests {let format = JsonFormat::default();let opt = ListingOptions::new(Arc::new(format)).with_collect_stat(collect_stat);let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]);- let table_path = ListingTableUrl::parse("test:///table/").unwrap();+ let table_path = ListingTableUrl::parse("test:///table/")?;let config = ListingTableConfig::new(table_path).with_listing_options(opt)diff --git a/datafusion/core/tests/catalog/memory.rs b/datafusion/core/tests/catalog/memory.rsindex ea9e71fc37..06ed141b2e 100644--- a/datafusion/core/tests/catalog/memory.rs+++ b/datafusion/core/tests/catalog/memory.rs@@ -19,7 +19,7 @@ use arrow::datatypes::Schema;use datafusion::catalog::CatalogProvider;use datafusion::datasource::empty::EmptyTable;use datafusion::datasource::listing::{- ListingTable, ListingTableConfig, ListingTableUrl,+ ListingTable, ListingTableConfig, ListingTableConfigExt, ListingTableUrl,};use datafusion::prelude::SessionContext;use datafusion_catalog::memory::*;diff --git a/datafusion/core/tests/parquet/schema_adapter.rs b/datafusion/core/tests/parquet/schema_adapter.rsindex 4ae2fa9b4c..40fc6176e2 100644--- a/datafusion/core/tests/parquet/schema_adapter.rs+++ b/datafusion/core/tests/parquet/schema_adapter.rs@@ -23,7 +23,9 @@ use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion::assert_batches_eq;use datafusion::common::Result;-use datafusion::datasource::listing::{ListingTable, ListingTableConfig};+use datafusion::datasource::listing::{+ ListingTable, ListingTableConfig, ListingTableConfigExt,+};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::DataFusionError;