Apache DataFusion SQL Query Engine
Remove SchemaAdapter (#19345)
Adrian Garcia Badaracco75d2473ba76c
diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rsindex 227cea5e7c..81d74cfbec 100644--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs@@ -63,9 +63,8 @@ const DEFAULT_VALUE_METADATA_KEY: &str = "example.default_value";/// 4. Use the DefaultPhysicalExprAdapter as a fallback for standard schema adaptation/// 5. Convert string default values to proper types using `ScalarValue::cast_to()` at planning time///-/// Important: PhysicalExprAdapter is specifically designed for rewriting filter predicates-/// that get pushed down to file scans. For handling missing columns in projections,-/// other mechanisms in DataFusion are used (like SchemaAdapter).+/// Important: PhysicalExprAdapter handles rewriting both filter predicates and projection+/// expressions for file scans, including handling missing columns.////// The metadata-based approach provides a flexible way to store default values as strings/// and cast them to the appropriate types at planning time, avoiding runtime overhead.@@ -144,8 +143,6 @@ pub async fn default_column_values() -> Result<()> {);println!("4. Default values from metadata are cast to proper types at planning time");println!("5. The DefaultPhysicalExprAdapter handles other schema adaptations");- println!("\nNote: PhysicalExprAdapter is specifically for filter predicates.");- println!("For projection columns, different mechanisms handle missing columns.");Ok(())}diff --git a/datafusion-examples/examples/data_io/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rsindex 2afc6c1047..e11a303f44 100644--- a/datafusion-examples/examples/data_io/parquet_index.rs+++ b/datafusion-examples/examples/data_io/parquet_index.rs@@ -511,7 +511,7 @@ impl ParquetMetadataIndexBuilder {// Get the schema of the file. A real system might have to handle the// case where the schema of the file is not the same as the schema of- // the other files e.g. using SchemaAdapter.+ // the other files e.g. using PhysicalExprAdapterFactory.if self.file_schema.is_none() {self.file_schema = Some(reader.schema().clone());}diff --git a/datafusion/catalog-listing/src/config.rs b/datafusion/catalog-listing/src/config.rsindex e3cd01a191..ca4d2abfcd 100644--- a/datafusion/catalog-listing/src/config.rs+++ b/datafusion/catalog-listing/src/config.rs@@ -21,6 +21,7 @@ use datafusion_catalog::Session;use datafusion_common::{config_err, internal_err};use datafusion_datasource::ListingTableUrl;use datafusion_datasource::file_compression_type::FileCompressionType;+#[expect(deprecated)]use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;use std::str::FromStr;@@ -44,15 +45,12 @@ pub enum SchemaSource {/// # Schema Evolution Support////// This configuration supports schema evolution through the optional-/// [`SchemaAdapterFactory`]. You might want to override the default factory when you need:+/// [`PhysicalExprAdapterFactory`]. You might want to override the default factory when you need:////// - **Type coercion requirements**: When you need custom logic for converting between/// different Arrow data types (e.g., Int32 ↔ Int64, Utf8 ↔ LargeUtf8)/// - **Column mapping**: You need to map columns with a legacy name to a new name/// - **Custom handling of missing columns**: By default they are filled in with nulls, but you may e.g. want to fill them in with `0` or `""`.-///-/// If not specified, a [`datafusion_datasource::schema_adapter::DefaultSchemaAdapterFactory`]-/// will be used, which handles basic schema compatibility cases.#[derive(Debug, Clone, Default)]pub struct ListingTableConfig {/// Paths on the `ObjectStore` for creating [`crate::ListingTable`].@@ -68,8 +66,6 @@ pub struct ListingTableConfig {pub options: Option<ListingOptions>,/// Tracks the source of the schema informationpub(crate) schema_source: SchemaSource,- /// Optional [`SchemaAdapterFactory`] for creating schema adapters- pub(crate) schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,/// Optional [`PhysicalExprAdapterFactory`] for creating physical expression adapterspub(crate) expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,}@@ -218,8 +214,7 @@ impl ListingTableConfig {file_schema,options: _,schema_source,- schema_adapter_factory,- expr_adapter_factory: physical_expr_adapter_factory,+ expr_adapter_factory,} = self;let (schema, new_schema_source) = match file_schema {@@ -241,8 +236,7 @@ impl ListingTableConfig {file_schema: Some(schema),options: Some(options),schema_source: new_schema_source,- schema_adapter_factory,- expr_adapter_factory: physical_expr_adapter_factory,+ expr_adapter_factory,})}None => internal_err!("No `ListingOptions` set for inferring schema"),@@ -282,7 +276,6 @@ impl ListingTableConfig {file_schema: self.file_schema,options: Some(options),schema_source: self.schema_source,- schema_adapter_factory: self.schema_adapter_factory,expr_adapter_factory: self.expr_adapter_factory,})}@@ -290,63 +283,11 @@ impl ListingTableConfig {}}- /// Set the [`SchemaAdapterFactory`] for the [`crate::ListingTable`]- ///- /// The schema adapter factory is used to create schema adapters that can- /// handle schema evolution and type conversions when reading files with- /// different schemas than the table schema.- ///- /// If not provided, a default schema adapter factory will be used.- ///- /// # Example: Custom Schema Adapter for Type Coercion- /// ```rust- /// # use std::sync::Arc;- /// # use datafusion_catalog_listing::{ListingTableConfig, ListingOptions};- /// # use datafusion_datasource::schema_adapter::{SchemaAdapterFactory, SchemaAdapter};- /// # use datafusion_datasource::ListingTableUrl;- /// # use datafusion_datasource_parquet::file_format::ParquetFormat;- /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType};- /// #- /// # #[derive(Debug)]- /// # struct MySchemaAdapterFactory;- /// # impl SchemaAdapterFactory for MySchemaAdapterFactory {- /// # fn create(&self, _projected_table_schema: SchemaRef, _file_schema: SchemaRef) -> Box<dyn SchemaAdapter> {- /// # unimplemented!()- /// # }- /// # }- /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap();- /// # let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default()));- /// # let table_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));- /// let config = ListingTableConfig::new(table_paths)- /// .with_listing_options(listing_options)- /// .with_schema(table_schema)- /// .with_schema_adapter_factory(Arc::new(MySchemaAdapterFactory));- /// ```- pub fn with_schema_adapter_factory(- self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Self {- Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self- }- }-- /// Get the [`SchemaAdapterFactory`] for this configuration- pub fn schema_adapter_factory(&self) -> Option<&Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.as_ref()- }-/// Set the [`PhysicalExprAdapterFactory`] for the [`crate::ListingTable`]////// The expression adapter factory is used to create physical expression adapters that can/// handle schema evolution and type conversions when evaluating expressions/// with different schemas than the table schema.- ///- /// If not provided, a default physical expression adapter factory will be used unless a custom- /// `SchemaAdapterFactory` is set, in which case only the `SchemaAdapterFactory` will be used.- ///- /// See <https://github.com/apache/datafusion/issues/16800> for details on this transition.pub fn with_expr_adapter_factory(self,expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,@@ -356,4 +297,23 @@ impl ListingTableConfig {..self}}++ /// Deprecated: Set the [`SchemaAdapterFactory`] for the [`crate::ListingTable`]+ ///+ /// `SchemaAdapterFactory` has been removed. Use [`Self::with_expr_adapter_factory`]+ /// and `PhysicalExprAdapterFactory` instead. See `upgrading.md` for more details.+ ///+ /// This method is a no-op and returns `self` unchanged.+ #[deprecated(+ since = "52.0.0",+ note = "SchemaAdapterFactory has been removed. Use with_expr_adapter_factory and PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ )]+ #[expect(deprecated)]+ pub fn with_schema_adapter_factory(+ self,+ _schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,+ ) -> Self {+ // No-op - just return self unchanged+ self+ }}diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 1dbfaf381a..9fb2dd2dce 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -29,6 +29,7 @@ use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};use datafusion_datasource::file_sink_config::FileSinkConfig;+#[expect(deprecated)]use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_datasource::{ListingTableUrl, PartitionedFile, TableSchema, compute_all_files_statistics,@@ -191,8 +192,6 @@ pub struct ListingTable {constraints: Constraints,/// Column default expressions for columns that are not physically present in the data filescolumn_defaults: HashMap<String, Expr>,- /// Optional [`SchemaAdapterFactory`] for creating schema adapters- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,/// Optional [`PhysicalExprAdapterFactory`] for creating physical expression adaptersexpr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,}@@ -235,7 +234,6 @@ impl ListingTable {collected_statistics: Arc::new(DefaultFileStatisticsCache::default()),constraints: Constraints::default(),column_defaults: HashMap::new(),- schema_adapter_factory: config.schema_adapter_factory,expr_adapter_factory: config.expr_adapter_factory,};@@ -290,48 +288,42 @@ impl ListingTable {self.schema_source}- /// Set the [`SchemaAdapterFactory`] for this [`ListingTable`]+ /// Deprecated: Set the [`SchemaAdapterFactory`] for this [`ListingTable`]///- /// The schema adapter factory is used to create schema adapters that can- /// handle schema evolution and type conversions when reading files with- /// different schemas than the table schema.+ /// `SchemaAdapterFactory` has been removed. Use [`ListingTableConfig::with_expr_adapter_factory`]+ /// and `PhysicalExprAdapterFactory` instead. See `upgrading.md` for more details.///- /// # Example: Adding Schema Evolution Support- /// ```rust- /// # use std::sync::Arc;- /// # use datafusion_catalog_listing::{ListingTable, ListingTableConfig, ListingOptions};- /// # use datafusion_datasource::ListingTableUrl;- /// # use datafusion_datasource::schema_adapter::{DefaultSchemaAdapterFactory, SchemaAdapter};- /// # use datafusion_datasource_parquet::file_format::ParquetFormat;- /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType};- /// # let table_path = ListingTableUrl::parse("file:///path/to/data").unwrap();- /// # let options = ListingOptions::new(Arc::new(ParquetFormat::default()));- /// # let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));- /// # let config = ListingTableConfig::new(table_path).with_listing_options(options).with_schema(schema);- /// # let table = ListingTable::try_new(config).unwrap();- /// let table_with_evolution = table- /// .with_schema_adapter_factory(Arc::new(DefaultSchemaAdapterFactory));- /// ```- /// See [`ListingTableConfig::with_schema_adapter_factory`] for an example of custom SchemaAdapterFactory.+ /// This method is a no-op and returns `self` unchanged.+ #[deprecated(+ since = "52.0.0",+ note = "SchemaAdapterFactory has been removed. Use ListingTableConfig::with_expr_adapter_factory and PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ )]+ #[expect(deprecated)]pub fn with_schema_adapter_factory(self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,+ _schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,) -> Self {- Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self- }+ // No-op - just return self unchanged+ self}- /// Get the [`SchemaAdapterFactory`] for this table- pub fn schema_adapter_factory(&self) -> Option<&Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.as_ref()+ /// Deprecated: Returns the [`SchemaAdapterFactory`] used by this [`ListingTable`].+ ///+ /// `SchemaAdapterFactory` has been removed. Use `PhysicalExprAdapterFactory` instead.+ /// See `upgrading.md` for more details.+ ///+ /// Always returns `None`.+ #[deprecated(+ since = "52.0.0",+ note = "SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ )]+ #[expect(deprecated)]+ pub fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {+ None}- /// Creates a file source and applies schema adapter factory if available- fn create_file_source_with_schema_adapter(- &self,- ) -> datafusion_common::Result<Arc<dyn FileSource>> {+ /// Creates a file source for this table+ fn create_file_source(&self) -> Arc<dyn FileSource> {let table_schema = TableSchema::new(Arc::clone(&self.file_schema),self.options@@ -341,13 +333,7 @@ impl ListingTable {.collect(),);- let mut source = self.options.format.file_source(table_schema);- // Apply schema adapter to source if available.- // The source will use this SchemaAdapter to adapt data batches as they flow up the plan.- if let Some(factory) = &self.schema_adapter_factory {- source = source.with_schema_adapter_factory(Arc::clone(factory))?;- }- Ok(source)+ self.options.format.file_source(table_schema)}/// If file_sort_order is specified, creates the appropriate physical expressions@@ -490,7 +476,7 @@ impl TableProvider for ListingTable {)))));};- let file_source = self.create_file_source_with_schema_adapter()?;+ let file_source = self.create_file_source();// create the execution planlet plan = selfdiff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex e205cec115..93d77e10ba 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -1453,11 +1453,10 @@ mod tests {}#[tokio::test]- async fn test_statistics_mapping_with_default_factory() -> Result<()> {+ async fn test_basic_table_scan() -> Result<()> {let ctx = SessionContext::new();- // Create a table without providing a custom schema adapter factory- // This should fall back to using DefaultSchemaAdapterFactory+ // Test basic table creation and scanninglet path = "table/file.json";register_test_store(&ctx, &[(path, 10)]);@@ -1469,25 +1468,18 @@ mod tests {let config = ListingTableConfig::new(table_path).with_listing_options(opt).with_schema(Arc::new(schema));- // Note: NOT calling .with_schema_adapter_factory() to test default behaviorlet table = ListingTable::try_new(config)?;- // Verify that no custom schema adapter factory is set- assert!(table.schema_adapter_factory().is_none());-- // The scan should work correctly with the default schema adapter+ // The scan should work correctlylet scan_result = table.scan(&ctx.state(), None, &[], None).await;- assert!(- scan_result.is_ok(),- "Scan should succeed with default schema adapter"- );+ assert!(scan_result.is_ok(), "Scan should succeed");- // Verify that the default adapter handles basic schema compatibility+ // Verify file listing workslet result = table.list_files_for_scan(&ctx.state(), &[], None).await?;assert!(!result.file_groups.is_empty(),- "Should list files successfully with default adapter"+ "Should list files successfully");Ok(())diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rsindex a6989a3cc5..aefda64d39 100644--- a/datafusion/core/src/datasource/mod.rs+++ b/datafusion/core/src/datasource/mod.rs@@ -60,13 +60,12 @@ mod tests {record_batch::RecordBatch,};use datafusion_common::{- Result, ScalarValue, record_batch,+ Result, ScalarValue,test_util::batches_to_sort_string,tree_node::{Transformed, TransformedResult, TreeNode},};use datafusion_datasource::{- PartitionedFile, file_scan_config::FileScanConfigBuilder,- schema_adapter::DefaultSchemaAdapterFactory, source::DataSourceExec,+ PartitionedFile, file_scan_config::FileScanConfigBuilder, source::DataSourceExec,};use datafusion_datasource_parquet::source::ParquetSource;use datafusion_physical_expr::expressions::{Column, Literal};@@ -149,58 +148,6 @@ mod tests {");}- #[test]- fn default_schema_adapter() {- let table_schema = Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- ]);-- // file has a subset of the table schema fields and different type- let file_schema = Schema::new(vec![- Field::new("c", DataType::Float64, true), // not in table schema- Field::new("b", DataType::Float64, true),- ]);-- let adapter = DefaultSchemaAdapterFactory::from_schema(Arc::new(table_schema));- let (mapper, indices) = adapter.map_schema(&file_schema).unwrap();- assert_eq!(indices, vec![1]);-- let file_batch = record_batch!(("b", Float64, vec![1.0, 2.0])).unwrap();-- let mapped_batch = mapper.map_batch(file_batch).unwrap();-- // the mapped batch has the correct schema and the "b" column has been cast to Utf8- let expected_batch = record_batch!(- ("a", Int32, vec![None, None]), // missing column filled with nulls- ("b", Utf8, vec!["1.0", "2.0"]) // b was cast to string and order was changed- )- .unwrap();- assert_eq!(mapped_batch, expected_batch);- }-- #[test]- fn default_schema_adapter_non_nullable_columns() {- let table_schema = Schema::new(vec![- Field::new("a", DataType::Int32, false), // "a"" is declared non nullable- Field::new("b", DataType::Utf8, true),- ]);- let file_schema = Schema::new(vec![- // since file doesn't have "a" it will be filled with nulls- Field::new("b", DataType::Float64, true),- ]);-- let adapter = DefaultSchemaAdapterFactory::from_schema(Arc::new(table_schema));- let (mapper, indices) = adapter.map_schema(&file_schema).unwrap();- assert_eq!(indices, vec![0]);-- let file_batch = record_batch!(("b", Float64, vec![1.0, 2.0])).unwrap();-- // Mapping fails because it tries to fill in a non-nullable column with nulls- let err = mapper.map_batch(file_batch).unwrap_err().to_string();- assert!(err.contains("Invalid argument error: Column 'a' is declared as non-nullable but contains null values"), "{err}");- }-#[derive(Debug)]struct TestPhysicalExprAdapterFactory;diff --git a/datafusion/core/src/datasource/physical_plan/mod.rs b/datafusion/core/src/datasource/physical_plan/mod.rsindex c57b08545b..04c8ea129d 100644--- a/datafusion/core/src/datasource/physical_plan/mod.rs+++ b/datafusion/core/src/datasource/physical_plan/mod.rs@@ -51,138 +51,3 @@ pub use datafusion_datasource::file_sink_config::*;pub use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener, FileStream, OnError,};--#[cfg(test)]-mod tests {- use std::sync::Arc;-- use arrow::array::{- BinaryArray, BooleanArray, Float32Array, Int32Array, Int64Array, RecordBatch,- StringArray, UInt64Array,- cast::AsArray,- types::{Float32Type, Float64Type, UInt32Type},- };- use arrow::datatypes::{DataType, Field, Schema};- use arrow_schema::SchemaRef;-- use crate::datasource::schema_adapter::{- DefaultSchemaAdapterFactory, SchemaAdapterFactory,- };-- #[test]- fn schema_mapping_map_batch() {- let table_schema = Arc::new(Schema::new(vec![- Field::new("c1", DataType::Utf8, true),- Field::new("c2", DataType::UInt32, true),- Field::new("c3", DataType::Float64, true),- ]));-- let adapter = DefaultSchemaAdapterFactory- .create(table_schema.clone(), table_schema.clone());-- let file_schema = Schema::new(vec![- Field::new("c1", DataType::Utf8, true),- Field::new("c2", DataType::UInt64, true),- Field::new("c3", DataType::Float32, true),- ]);-- let (mapping, _) = adapter.map_schema(&file_schema).expect("map schema failed");-- let c1 = StringArray::from(vec!["hello", "world"]);- let c2 = UInt64Array::from(vec![9_u64, 5_u64]);- let c3 = Float32Array::from(vec![2.0_f32, 7.0_f32]);- let batch = RecordBatch::try_new(- Arc::new(file_schema),- vec![Arc::new(c1), Arc::new(c2), Arc::new(c3)],- )- .unwrap();-- let mapped_batch = mapping.map_batch(batch).unwrap();-- assert_eq!(mapped_batch.schema(), table_schema);- assert_eq!(mapped_batch.num_columns(), 3);- assert_eq!(mapped_batch.num_rows(), 2);-- let c1 = mapped_batch.column(0).as_string::<i32>();- let c2 = mapped_batch.column(1).as_primitive::<UInt32Type>();- let c3 = mapped_batch.column(2).as_primitive::<Float64Type>();-- assert_eq!(c1.value(0), "hello");- assert_eq!(c1.value(1), "world");- assert_eq!(c2.value(0), 9_u32);- assert_eq!(c2.value(1), 5_u32);- assert_eq!(c3.value(0), 2.0_f64);- assert_eq!(c3.value(1), 7.0_f64);- }-- #[test]- fn schema_adapter_map_schema_with_projection() {- let table_schema = Arc::new(Schema::new(vec![- Field::new("c0", DataType::Utf8, true),- Field::new("c1", DataType::Utf8, true),- Field::new("c2", DataType::Float64, true),- Field::new("c3", DataType::Int32, true),- Field::new("c4", DataType::Float32, true),- ]));-- let file_schema = Schema::new(vec![- Field::new("id", DataType::Int32, true),- Field::new("c1", DataType::Boolean, true),- Field::new("c2", DataType::Float32, true),- Field::new("c3", DataType::Binary, true),- Field::new("c4", DataType::Int64, true),- ]);-- let indices = vec![1, 2, 4];- let schema = SchemaRef::from(table_schema.project(&indices).unwrap());- let adapter = DefaultSchemaAdapterFactory.create(schema, table_schema.clone());- let (mapping, projection) = adapter.map_schema(&file_schema).unwrap();-- let id = Int32Array::from(vec![Some(1), Some(2), Some(3)]);- let c1 = BooleanArray::from(vec![Some(true), Some(false), Some(true)]);- let c2 = Float32Array::from(vec![Some(2.0_f32), Some(7.0_f32), Some(3.0_f32)]);- let c3 = BinaryArray::from_opt_vec(vec![- Some(b"hallo"),- Some(b"danke"),- Some(b"super"),- ]);- let c4 = Int64Array::from(vec![1, 2, 3]);- let batch = RecordBatch::try_new(- Arc::new(file_schema),- vec![- Arc::new(id),- Arc::new(c1),- Arc::new(c2),- Arc::new(c3),- Arc::new(c4),- ],- )- .unwrap();- let rows_num = batch.num_rows();- let projected = batch.project(&projection).unwrap();- let mapped_batch = mapping.map_batch(projected).unwrap();-- assert_eq!(- mapped_batch.schema(),- Arc::new(table_schema.project(&indices).unwrap())- );- assert_eq!(mapped_batch.num_columns(), indices.len());- assert_eq!(mapped_batch.num_rows(), rows_num);-- let c1 = mapped_batch.column(0).as_string::<i32>();- let c2 = mapped_batch.column(1).as_primitive::<Float64Type>();- let c4 = mapped_batch.column(2).as_primitive::<Float32Type>();-- assert_eq!(c1.value(0), "true");- assert_eq!(c1.value(1), "false");- assert_eq!(c1.value(2), "true");-- assert_eq!(c2.value(0), 2.0_f64);- assert_eq!(c2.value(1), 7.0_f64);- assert_eq!(c2.value(2), 3.0_f64);-- assert_eq!(c4.value(0), 1.0_f32);- assert_eq!(c4.value(1), 2.0_f32);- assert_eq!(c4.value(2), 3.0_f32);- }-}diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rsindex abcd6bcf2e..4703b55ecc 100644--- a/datafusion/core/src/datasource/physical_plan/parquet.rs+++ b/datafusion/core/src/datasource/physical_plan/parquet.rs@@ -306,7 +306,7 @@ mod tests {let batch = RecordBatch::try_new(file_schema.clone(), vec![c1]).unwrap();- // Since c2 is missing from the file and we didn't supply a custom `SchemaAdapterFactory`,+ // Since c2 is missing from the file and we didn't supply a custom `PhysicalExprAdapterFactory`,// the default behavior is to fill in missing columns with nulls.// Thus this predicate will come back as false.let filter = col("c2").eq(lit(1_i32));@@ -364,7 +364,7 @@ mod tests {let batch = RecordBatch::try_new(file_schema.clone(), vec![c1]).unwrap();- // Since c2 is missing from the file and we didn't supply a custom `SchemaAdapterFactory`,+ // Since c2 is missing from the file and we didn't supply a custom `PhysicalExprAdapterFactory`,// the default behavior is to fill in missing columns with nulls.// Thus this predicate will come back as false.let filter = col("c2").eq(lit("abc"));@@ -426,7 +426,7 @@ mod tests {let batch = RecordBatch::try_new(file_schema.clone(), vec![c1, c3]).unwrap();- // Since c2 is missing from the file and we didn't supply a custom `SchemaAdapterFactory`,+ // Since c2 is missing from the file and we didn't supply a custom `PhysicalExprAdapterFactory`,// the default behavior is to fill in missing columns with nulls.// Thus this predicate will come back as false.let filter = col("c2").eq(lit("abc"));@@ -488,7 +488,7 @@ mod tests {let batch =RecordBatch::try_new(file_schema.clone(), vec![c3.clone(), c3]).unwrap();- // Since c2 is missing from the file and we didn't supply a custom `SchemaAdapterFactory`,+ // Since c2 is missing from the file and we didn't supply a custom `PhysicalExprAdapterFactory`,// the default behavior is to fill in missing columns with nulls.// Thus this predicate will come back as false.let filter = col("c2").eq(lit("abc"));@@ -1324,7 +1324,7 @@ mod tests {async fn parquet_exec_with_int96_from_spark() -> Result<()> {// arrow-rs relies on the chrono library to convert between timestamps and strings, so// instead compare as Int64. The underlying type should be a PrimitiveArray of Int64- // anyway, so this should be a zero-copy non-modifying cast at the SchemaAdapter.+ // anyway, so this should be a zero-copy non-modifying cast.let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, true)]));let testdata = datafusion_common::test_util::parquet_test_data();diff --git a/datafusion/core/tests/core_integration.rs b/datafusion/core/tests/core_integration.rsindex cc4dfcf720..bdbe722453 100644--- a/datafusion/core/tests/core_integration.rs+++ b/datafusion/core/tests/core_integration.rs@@ -48,9 +48,6 @@ mod optimizer;/// Run all tests that are found in the `physical_optimizer` directorymod physical_optimizer;-/// Run all tests that are found in the `schema_adapter` directory-mod schema_adapter;-/// Run all tests that are found in the `serde` directorymod serde;diff --git a/datafusion/core/tests/parquet/schema_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rssimilarity index 70%rename from datafusion/core/tests/parquet/schema_adapter.rsrename to datafusion/core/tests/parquet/expr_adapter.rsindex 7e2240cf6b..515422ed75 100644--- a/datafusion/core/tests/parquet/schema_adapter.rs+++ b/datafusion/core/tests/parquet/expr_adapter.rs@@ -221,11 +221,8 @@ async fn test_custom_schema_adapter_and_custom_expression_adapter() {/// Test demonstrating how to implement a custom PhysicalExprAdapterFactory/// that fills missing columns with non-null default values.///-/// This is the recommended migration path for users who previously used-/// SchemaAdapterFactory to fill missing columns with default values.-/// Instead of transforming batches after reading (SchemaAdapter::map_batch),-/// the PhysicalExprAdapterFactory rewrites expressions to use literals for-/// missing columns, achieving the same result more efficiently.+/// PhysicalExprAdapterFactory rewrites expressions to use literals for+/// missing columns, handling schema evolution efficiently at planning time.#[tokio::test]async fn test_physical_expr_adapter_with_non_null_defaults() {// File only has c1 column@@ -322,3 +319,148 @@ async fn test_physical_expr_adapter_with_non_null_defaults() {];assert_batches_eq!(expected, &batches);}++/// Test demonstrating that a single PhysicalExprAdapterFactory instance can be+/// reused across multiple ListingTable instances.+///+/// This addresses the concern: "This is important for ListingTable. A test for+/// ListingTable would add assurance that the functionality is retained [i.e. we+/// can re-use a PhysicalExprAdapterFactory]"+#[tokio::test]+async fn test_physical_expr_adapter_factory_reuse_across_tables() {+ // Create two different parquet files with different schemas+ // File 1: has column c1 only+ let batch1 = record_batch!(("c1", Int32, [1, 2, 3])).unwrap();+ // File 2: has column c1 only but different data+ let batch2 = record_batch!(("c1", Int32, [10, 20, 30])).unwrap();++ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ let store_url = ObjectStoreUrl::parse("memory://").unwrap();++ // Write files to different paths+ write_parquet(batch1, store.clone(), "table1/data.parquet").await;+ write_parquet(batch2, store.clone(), "table2/data.parquet").await;++ // Table schema has additional columns that don't exist in files+ let table_schema = Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int64, false),+ Field::new("c2", DataType::Utf8, true), // missing from files+ ]));++ let mut cfg = SessionConfig::new()+ .with_collect_statistics(false)+ .with_parquet_pruning(false);+ cfg.options_mut().execution.parquet.pushdown_filters = true;+ let ctx = SessionContext::new_with_config(cfg);+ ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));++ // Create ONE factory instance wrapped in Arc - this will be REUSED+ let factory: Arc<dyn PhysicalExprAdapterFactory> =+ Arc::new(CustomPhysicalExprAdapterFactory);++ // Create ListingTable 1 using the shared factory+ let listing_table_config1 =+ ListingTableConfig::new(ListingTableUrl::parse("memory:///table1/").unwrap())+ .infer_options(&ctx.state())+ .await+ .unwrap()+ .with_schema(table_schema.clone())+ .with_expr_adapter_factory(Arc::clone(&factory)); // Clone the Arc, not create new factory++ let table1 = ListingTable::try_new(listing_table_config1).unwrap();+ ctx.register_table("t1", Arc::new(table1)).unwrap();++ // Create ListingTable 2 using the SAME factory instance+ let listing_table_config2 =+ ListingTableConfig::new(ListingTableUrl::parse("memory:///table2/").unwrap())+ .infer_options(&ctx.state())+ .await+ .unwrap()+ .with_schema(table_schema.clone())+ .with_expr_adapter_factory(Arc::clone(&factory)); // Reuse same factory++ let table2 = ListingTable::try_new(listing_table_config2).unwrap();+ ctx.register_table("t2", Arc::new(table2)).unwrap();++ // Verify table 1 works correctly with the shared factory+ // CustomPhysicalExprAdapterFactory fills missing Utf8 columns with 'b'+ let batches = ctx+ .sql("SELECT c1, c2 FROM t1 ORDER BY c1")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();++ let expected = [+ "+----+----+",+ "| c1 | c2 |",+ "+----+----+",+ "| 1 | b |",+ "| 2 | b |",+ "| 3 | b |",+ "+----+----+",+ ];+ assert_batches_eq!(expected, &batches);++ // Verify table 2 also works correctly with the SAME shared factory+ let batches = ctx+ .sql("SELECT c1, c2 FROM t2 ORDER BY c1")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();++ let expected = [+ "+----+----+",+ "| c1 | c2 |",+ "+----+----+",+ "| 10 | b |",+ "| 20 | b |",+ "| 30 | b |",+ "+----+----+",+ ];+ assert_batches_eq!(expected, &batches);++ // Verify predicates work on both tables with the shared factory+ let batches = ctx+ .sql("SELECT c1 FROM t1 WHERE c2 = 'b' ORDER BY c1")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();++ #[rustfmt::skip]+ let expected = [+ "+----+",+ "| c1 |",+ "+----+",+ "| 1 |",+ "| 2 |",+ "| 3 |",+ "+----+",+ ];+ assert_batches_eq!(expected, &batches);++ let batches = ctx+ .sql("SELECT c1 FROM t2 WHERE c2 = 'b' ORDER BY c1")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();++ #[rustfmt::skip]+ let expected = [+ "+----+",+ "| c1 |",+ "+----+",+ "| 10 |",+ "| 20 |",+ "| 30 |",+ "+----+",+ ];+ assert_batches_eq!(expected, &batches);+}diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rsindex 0a0478dd03..35b5918d9e 100644--- a/datafusion/core/tests/parquet/mod.rs+++ b/datafusion/core/tests/parquet/mod.rs@@ -46,13 +46,13 @@ use tempfile::NamedTempFile;mod custom_reader;#[cfg(feature = "parquet_encryption")]mod encryption;+mod expr_adapter;mod external_access_plan;mod file_statistics;mod filter_pushdown;mod page_pruning;mod row_group_pruning;mod schema;-mod schema_adapter;mod schema_coercion;mod utils;diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rsindex 93e928b51a..1afdc4823f 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs@@ -22,8 +22,7 @@ use datafusion_common::{Result, config::ConfigOptions, internal_err};use datafusion_datasource::{PartitionedFile, file::FileSource, file_scan_config::FileScanConfig,file_scan_config::FileScanConfigBuilder, file_stream::FileOpenFuture,- file_stream::FileOpener, schema_adapter::DefaultSchemaAdapterFactory,- schema_adapter::SchemaAdapterFactory, source::DataSourceExec,+ file_stream::FileOpener, source::DataSourceExec,};use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_optimizer::PhysicalOptimizerRule;@@ -51,7 +50,6 @@ use std::{pub struct TestOpener {batches: Vec<RecordBatch>,batch_size: Option<usize>,- schema: SchemaRef,projection: Option<Vec<usize>>,predicate: Option<Arc<dyn PhysicalExpr>>,}@@ -73,8 +71,6 @@ impl FileOpener for TestOpener {batches = new_batches.into_iter().collect();}- let factory = DefaultSchemaAdapterFactory::from_schema(Arc::clone(&self.schema));- let (mapper, projection) = factory.map_schema(&batches[0].schema()).unwrap();let mut new_batches = Vec::new();for batch in batches {let batch = if let Some(predicate) = &self.predicate {@@ -82,9 +78,6 @@ impl FileOpener for TestOpener {} else {batch};-- let batch = batch.project(&projection).unwrap();- let batch = mapper.map_batch(batch).unwrap();new_batches.push(batch);}batches = new_batches;@@ -109,10 +102,8 @@ pub struct TestSource {predicate: Option<Arc<dyn PhysicalExpr>>,batch_size: Option<usize>,batches: Vec<RecordBatch>,- schema: SchemaRef,metrics: ExecutionPlanMetricsSet,projection: Option<Vec<usize>>,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,table_schema: datafusion_datasource::TableSchema,}@@ -121,14 +112,12 @@ impl TestSource {let table_schema =datafusion_datasource::TableSchema::new(Arc::clone(&schema), vec![]);Self {- schema,support,metrics: ExecutionPlanMetricsSet::new(),batches,predicate: None,batch_size: None,projection: None,- schema_adapter_factory: None,table_schema,}}@@ -144,7 +133,6 @@ impl FileSource for TestSource {Ok(Arc::new(TestOpener {batches: self.batches.clone(),batch_size: self.batch_size,- schema: Arc::clone(&self.schema),projection: self.projection.clone(),predicate: self.predicate.clone(),}))@@ -222,20 +210,6 @@ impl FileSource for TestSource {}}- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }-fn table_schema(&self) -> &datafusion_datasource::TableSchema {&self.table_schema}diff --git a/datafusion/core/tests/schema_adapter/mod.rs b/datafusion/core/tests/schema_adapter/mod.rsdeleted file mode 100644index 2f81a43f47..0000000000--- a/datafusion/core/tests/schema_adapter/mod.rs+++ /dev/null@@ -1,18 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--mod schema_adapter_integration_tests;diff --git a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs b/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rsdeleted file mode 100644index 01242ff41f..0000000000--- a/datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs+++ /dev/null@@ -1,752 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--use std::sync::Arc;--use arrow::array::RecordBatch;--use arrow_schema::{DataType, Field, Schema, SchemaRef};-use bytes::{BufMut, BytesMut};-use datafusion::common::Result;-use datafusion::config::{ConfigOptions, TableParquetOptions};-use datafusion::datasource::listing::PartitionedFile;-#[cfg(feature = "parquet")]-use datafusion::datasource::physical_plan::ParquetSource;-use datafusion::datasource::physical_plan::{- ArrowSource, CsvSource, FileSource, JsonSource,-};-use datafusion::logical_expr::{col, lit};-use datafusion::physical_plan::ExecutionPlan;-use datafusion::prelude::SessionContext;-use datafusion_common::ColumnStatistics;-use datafusion_common::config::CsvOptions;-use datafusion_common::record_batch;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_datasource::file_scan_config::FileScanConfigBuilder;-use datafusion_datasource::schema_adapter::{- SchemaAdapter, SchemaAdapterFactory, SchemaMapper,-};--use datafusion::assert_batches_eq;-use datafusion_datasource::TableSchema;-use datafusion_datasource::source::DataSourceExec;-use datafusion_execution::object_store::ObjectStoreUrl;-use datafusion_expr::Expr;-use datafusion_physical_expr::expressions::Column;-use datafusion_physical_expr::planner::logical2physical;-use datafusion_physical_expr::projection::ProjectionExprs;-use datafusion_physical_expr_adapter::{PhysicalExprAdapter, PhysicalExprAdapterFactory};-use datafusion_physical_expr_common::physical_expr::PhysicalExpr;-use object_store::{ObjectStore, memory::InMemory, path::Path};-use parquet::arrow::ArrowWriter;--async fn write_parquet(batch: RecordBatch, store: Arc<dyn ObjectStore>, path: &str) {- write_batches_to_parquet(&[batch], store, path).await;-}--/// Write RecordBatches to a Parquet file with each batch in its own row group.-async fn write_batches_to_parquet(- batches: &[RecordBatch],- store: Arc<dyn ObjectStore>,- path: &str,-) -> usize {- let mut out = BytesMut::new().writer();- {- let mut writer =- ArrowWriter::try_new(&mut out, batches[0].schema(), None).unwrap();- for batch in batches {- writer.write(batch).unwrap();- writer.flush().unwrap();- }- writer.finish().unwrap();- }- let data = out.into_inner().freeze();- let file_size = data.len();- store.put(&Path::from(path), data.into()).await.unwrap();- file_size-}--/// A schema adapter factory that transforms column names to uppercase-#[derive(Debug, PartialEq)]-struct UppercaseAdapterFactory {}--impl SchemaAdapterFactory for UppercaseAdapterFactory {- fn create(- &self,- projected_table_schema: SchemaRef,- _table_schema: SchemaRef,- ) -> Box<dyn SchemaAdapter> {- Box::new(UppercaseAdapter {- table_schema: projected_table_schema,- })- }-}--/// Schema adapter that transforms column names to uppercase-#[derive(Debug)]-struct UppercaseAdapter {- table_schema: SchemaRef,-}--impl SchemaAdapter for UppercaseAdapter {- fn map_column_index(&self, index: usize, file_schema: &Schema) -> Option<usize> {- let field = self.table_schema.field(index);- let uppercase_name = field.name().to_uppercase();- file_schema- .fields()- .iter()- .position(|f| f.name().to_uppercase() == uppercase_name)- }-- fn map_schema(- &self,- file_schema: &Schema,- ) -> Result<(Arc<dyn SchemaMapper>, Vec<usize>)> {- let mut projection = Vec::new();-- // Map each field in the table schema to the corresponding field in the file schema- for table_field in self.table_schema.fields() {- let uppercase_name = table_field.name().to_uppercase();- if let Some(pos) = file_schema- .fields()- .iter()- .position(|f| f.name().to_uppercase() == uppercase_name)- {- projection.push(pos);- }- }-- let mapper = UppercaseSchemaMapper {- output_schema: self.output_schema(),- projection: projection.clone(),- };-- Ok((Arc::new(mapper), projection))- }-}--impl UppercaseAdapter {- fn output_schema(&self) -> SchemaRef {- let fields: Vec<Field> = self- .table_schema- .fields()- .iter()- .map(|f| {- Field::new(- f.name().to_uppercase().as_str(),- f.data_type().clone(),- f.is_nullable(),- )- })- .collect();-- Arc::new(Schema::new(fields))- }-}--#[derive(Debug)]-struct UppercaseSchemaMapper {- output_schema: SchemaRef,- projection: Vec<usize>,-}--impl SchemaMapper for UppercaseSchemaMapper {- fn map_batch(&self, batch: RecordBatch) -> Result<RecordBatch> {- let columns = self- .projection- .iter()- .map(|&i| batch.column(i).clone())- .collect::<Vec<_>>();- Ok(RecordBatch::try_new(self.output_schema.clone(), columns)?)- }-- fn map_column_statistics(- &self,- stats: &[ColumnStatistics],- ) -> Result<Vec<ColumnStatistics>> {- Ok(self- .projection- .iter()- .map(|&i| stats.get(i).cloned().unwrap_or_default())- .collect())- }-}--/// A physical expression adapter factory that maps uppercase column names to lowercase-#[derive(Debug)]-struct UppercasePhysicalExprAdapterFactory;--impl PhysicalExprAdapterFactory for UppercasePhysicalExprAdapterFactory {- fn create(- &self,- _logical_file_schema: SchemaRef,- physical_file_schema: SchemaRef,- ) -> Arc<dyn PhysicalExprAdapter> {- Arc::new(UppercasePhysicalExprAdapter {- physical_file_schema,- })- }-}--#[derive(Debug)]-struct UppercasePhysicalExprAdapter {- physical_file_schema: SchemaRef,-}--impl PhysicalExprAdapter for UppercasePhysicalExprAdapter {- fn rewrite(&self, expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {- expr.transform(|e| {- if let Some(column) = e.as_any().downcast_ref::<Column>() {- // Map uppercase column name (from logical schema) to lowercase (in physical file)- let lowercase_name = column.name().to_lowercase();- if let Ok(idx) = self.physical_file_schema.index_of(&lowercase_name) {- return Ok(Transformed::yes(- Arc::new(Column::new(&lowercase_name, idx))- as Arc<dyn PhysicalExpr>,- ));- }- }- Ok(Transformed::no(e))- })- .data()- }-}--#[derive(Clone)]-struct ParquetTestCase {- table_schema: TableSchema,- batches: Vec<RecordBatch>,- predicate: Option<Expr>,- projection: Option<ProjectionExprs>,- push_down_filters: bool,-}--impl ParquetTestCase {- fn new(table_schema: TableSchema, batches: Vec<RecordBatch>) -> Self {- Self {- table_schema,- batches,- predicate: None,- projection: None,- push_down_filters: true,- }- }-- fn push_down_filters(mut self, pushdown_filters: bool) -> Self {- self.push_down_filters = pushdown_filters;- self- }-- fn with_predicate(mut self, predicate: Expr) -> Self {- self.predicate = Some(predicate);- self- }-- fn with_projection(mut self, projection: ProjectionExprs) -> Self {- self.projection = Some(projection);- self- }-- async fn execute(self) -> Result<Vec<RecordBatch>> {- let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;- let store_url = ObjectStoreUrl::parse("memory://").unwrap();- let path = "test.parquet";- let file_size =- write_batches_to_parquet(&self.batches, store.clone(), path).await;-- let ctx = SessionContext::new();- ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));-- let mut table_options = TableParquetOptions::default();- // controlled via ConfigOptions flag; ParquetSources ORs them so if either is true then pushdown is enabled- table_options.global.pushdown_filters = false;- let mut file_source = Arc::new(- ParquetSource::new(self.table_schema.table_schema().clone())- .with_table_parquet_options(table_options),- ) as Arc<dyn FileSource>;-- if let Some(projection) = self.projection {- file_source = file_source.try_pushdown_projection(&projection)?.unwrap();- }-- if let Some(predicate) = &self.predicate {- let filter_expr =- logical2physical(predicate, self.table_schema.table_schema());- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = self.push_down_filters;- let result = file_source.try_pushdown_filters(vec![filter_expr], &config)?;- file_source = result.updated_node.unwrap();- }-- let config = FileScanConfigBuilder::new(store_url.clone(), file_source)- .with_file(PartitionedFile::new(path, file_size as u64)) // size 0 for test- .with_expr_adapter(None)- .build();-- let exec = DataSourceExec::from_data_source(config);- let task_ctx = ctx.task_ctx();- let stream = exec.execute(0, task_ctx)?;- datafusion::physical_plan::common::collect(stream).await- }-}--/// Test reading and filtering a Parquet file where the table schema is flipped (c, b, a) vs. the physical file schema (a, b, c)-#[tokio::test]-#[cfg(feature = "parquet")]-async fn test_parquet_flipped_projection() -> Result<()> {- // Create test data with columns (a, b, c) - the file schema- let batch1 = record_batch!(- ("a", Int32, vec![1, 2]),- ("b", Utf8, vec!["x", "y"]),- ("c", Float64, vec![1.1, 2.2])- )?;- let batch2 = record_batch!(- ("a", Int32, vec![3]),- ("b", Utf8, vec!["z"]),- ("c", Float64, vec![3.3])- )?;-- // Create a table schema with flipped column order (c, b, a)- let table_schema = Arc::new(Schema::new(vec![- Field::new("c", DataType::Float64, false),- Field::new("b", DataType::Utf8, true),- Field::new("a", DataType::Int32, false),- ]));- let table_schema = TableSchema::from_file_schema(table_schema);-- let test_case = ParquetTestCase::new(table_schema.clone(), vec![batch1, batch2]);-- // Test reading with flipped schema- let batches = test_case.clone().execute().await?;- #[rustfmt::skip]- let expected = [- "+-----+---+---+",- "| c | b | a |",- "+-----+---+---+",- "| 1.1 | x | 1 |",- "| 2.2 | y | 2 |",- "| 3.3 | z | 3 |",- "+-----+---+---+",- ];- assert_batches_eq!(expected, &batches);-- // Test with a projection that selects (b, a)- let projection = ProjectionExprs::from_indices(&[1, 2], table_schema.table_schema());- let batches = test_case- .clone()- .with_projection(projection.clone())- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+---+---+",- "| b | a |",- "+---+---+",- "| x | 1 |",- "| y | 2 |",- "| z | 3 |",- "+---+---+",- ];- assert_batches_eq!(expected, &batches);-- // Test with a filter on b, a- // a = 1 or b != 'foo' and a = 3 -> matches [{a=1,b=x},{b=z,a=3}]- let filter = col("a")- .eq(lit(1))- .or(col("b").not_eq(lit("foo")).and(col("a").eq(lit(3))));- let batches = test_case- .clone()- .with_projection(projection.clone())- .with_predicate(filter.clone())- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+---+---+",- "| b | a |",- "+---+---+",- "| x | 1 |",- "| z | 3 |",- "+---+---+",- ];- assert_batches_eq!(expected, &batches);-- // Test with only statistics-based filter pushdown (no row-level filtering)- // Since we have 2 row groups and the filter matches rows in both, stats pruning alone won't filter any- let batches = test_case- .clone()- .with_projection(projection)- .with_predicate(filter)- .push_down_filters(false)- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+---+---+",- "| b | a |",- "+---+---+",- "| x | 1 |",- "| y | 2 |",- "| z | 3 |",- "+---+---+",- ];- assert_batches_eq!(expected, &batches);-- // Test with a filter that can prune via statistics: a > 10 (no rows match)- let filter = col("a").gt(lit(10));- let batches = test_case- .clone()- .with_predicate(filter)- .push_down_filters(false)- .execute()- .await?;- // Stats show a has max=3, so a > 10 prunes all row groups- assert_eq!(batches.len(), 0);-- // With a filter that matches only the first row group: a < 3- let filter = col("a").lt(lit(3));- let batches = test_case- .clone()- .with_predicate(filter)- .push_down_filters(false)- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+-----+---+---+",- "| c | b | a |",- "+-----+---+---+",- "| 1.1 | x | 1 |",- "| 2.2 | y | 2 |",- "+-----+---+---+",- ];- assert_batches_eq!(expected, &batches);-- Ok(())-}--/// Test reading a Parquet file that is missing a column specified in the table schema, which should get filled in with nulls by default.-/// We test with the file having columns (a, c) and the table schema having (a, b, c)-#[tokio::test]-#[cfg(feature = "parquet")]-async fn test_parquet_missing_column() -> Result<()> {- // Create test data with columns (a, c) as 2 batches- // | a | c |- // |---|-----|- // | 1 | 1.1 |- // | 2 | 2.2 |- // | ~ | ~~~ |- // | 3 | 3.3 |- let batch1 = record_batch!(("a", Int32, vec![1, 2]), ("c", Float64, vec![1.1, 2.2]))?;- let batch2 = record_batch!(("a", Int32, vec![3]), ("c", Float64, vec![3.3]))?;-- // Create a table schema with an extra column 'b' (a, b, c)- let logical_file_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, false),- Field::new("b", DataType::Utf8, true),- Field::new("c", DataType::Float64, false),- ]));- let table_schema = TableSchema::from_file_schema(logical_file_schema.clone());-- let test_case = ParquetTestCase::new(table_schema.clone(), vec![batch1, batch2]);-- let batches = test_case.clone().execute().await?;- #[rustfmt::skip]- let expected = [- "+---+---+-----+",- "| a | b | c |",- "+---+---+-----+",- "| 1 | | 1.1 |",- "| 2 | | 2.2 |",- "| 3 | | 3.3 |",- "+---+---+-----+",- ];- assert_batches_eq!(expected, &batches);-- // And with a projection applied that selects (`c, `a`, `b`)- let projection =- ProjectionExprs::from_indices(&[2, 0, 1], table_schema.table_schema());- let batches = test_case- .clone()- .with_projection(projection)- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+-----+---+---+",- "| c | a | b |",- "+-----+---+---+",- "| 1.1 | 1 | |",- "| 2.2 | 2 | |",- "| 3.3 | 3 | |",- "+-----+---+---+",- ];- assert_batches_eq!(expected, &batches);-- // And with a filter on a, b- // a = 1 or b is null and a = 3- let filter = col("a")- .eq(lit(1))- .or(col("b").is_null().and(col("a").eq(lit(3))));- let batches = test_case- .clone()- .with_predicate(filter.clone())- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+---+---+-----+",- "| a | b | c |",- "+---+---+-----+",- "| 1 | | 1.1 |",- "| 3 | | 3.3 |",- "+---+---+-----+",- ];- assert_batches_eq!(expected, &batches);- // With only statistics-based filter pushdown- let batches = test_case- .clone()- .with_predicate(filter)- .push_down_filters(false)- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+---+---+-----+",- "| a | b | c |",- "+---+---+-----+",- "| 1 | | 1.1 |",- "| 2 | | 2.2 |",- "| 3 | | 3.3 |",- "+---+---+-----+",- ];- assert_batches_eq!(expected, &batches);-- // Filter `b is not null or a = 24` doesn't match any rows- let filter = col("b").is_not_null().or(col("a").eq(lit(24)));- let batches = test_case- .clone()- .with_predicate(filter.clone())- .execute()- .await?;- // There should be zero batches- assert_eq!(batches.len(), 0);- // With only statistics-based filter pushdown- let batches = test_case- .clone()- .with_predicate(filter)- .push_down_filters(false)- .execute()- .await?;- // There should be zero batches- assert_eq!(batches.len(), 0);- // Check another filter: `b = 'foo' and a = 24` should also prune data with only statistics-based pushdown- let filter = col("b").eq(lit("foo")).and(col("a").eq(lit(24)));- let batches = test_case- .clone()- .with_predicate(filter)- .push_down_filters(false)- .execute()- .await?;- // There should be zero batches- assert_eq!(batches.len(), 0);- // On the other hand `b is null and a = 2` should prune only the second row group with stats only pruning- let filter = col("b").is_null().and(col("a").eq(lit(2)));- let batches = test_case- .clone()- .with_predicate(filter)- .push_down_filters(false)- .execute()- .await?;- #[rustfmt::skip]- let expected = [- "+---+---+-----+",- "| a | b | c |",- "+---+---+-----+",- "| 1 | | 1.1 |",- "| 2 | | 2.2 |",- "+---+---+-----+",- ];- assert_batches_eq!(expected, &batches);-- Ok(())-}--#[tokio::test]-#[cfg(feature = "parquet")]-async fn test_parquet_integration_with_physical_expr_adapter() -> Result<()> {- // Create test data- let batch = RecordBatch::try_new(- Arc::new(Schema::new(vec![- Field::new("id", DataType::Int32, false),- Field::new("name", DataType::Utf8, true),- ])),- vec![- Arc::new(arrow::array::Int32Array::from(vec![1, 2, 3])),- Arc::new(arrow::array::StringArray::from(vec!["a", "b", "c"])),- ],- )?;-- let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;- let store_url = ObjectStoreUrl::parse("memory://").unwrap();- let path = "test.parquet";- write_parquet(batch.clone(), store.clone(), path).await;-- // Get the actual file size from the object store- let object_meta = store.head(&Path::from(path)).await?;- let file_size = object_meta.size;-- // Create a session context and register the object store- let ctx = SessionContext::new();- ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));-- // Create a table schema with uppercase column names- let table_schema = Arc::new(Schema::new(vec![- Field::new("ID", DataType::Int32, false),- Field::new("NAME", DataType::Utf8, true),- ]));-- // Create a ParquetSource with the table schema (uppercase columns)- let file_source = Arc::new(ParquetSource::new(table_schema.clone()));-- // Use PhysicalExprAdapterFactory to map uppercase column names to lowercase- let config = FileScanConfigBuilder::new(store_url, file_source)- .with_file(PartitionedFile::new(path, file_size))- .with_expr_adapter(Some(Arc::new(UppercasePhysicalExprAdapterFactory)))- .build();-- // Create a data source executor- let exec = DataSourceExec::from_data_source(config);-- // Collect results- let task_ctx = ctx.task_ctx();- let stream = exec.execute(0, task_ctx)?;- let batches = datafusion::physical_plan::common::collect(stream).await?;-- // There should be one batch- assert_eq!(batches.len(), 1);-- // Verify the schema has the uppercase column names- let result_schema = batches[0].schema();- assert_eq!(result_schema.field(0).name(), "ID");- assert_eq!(result_schema.field(1).name(), "NAME");-- // Verify the data was correctly read from the lowercase file columns- // This confirms the PhysicalExprAdapter successfully mapped uppercase -> lowercase- let id_array = batches[0]- .column(0)- .as_any()- .downcast_ref::<arrow::array::Int32Array>()- .expect("Expected Int32Array for ID column");- assert_eq!(id_array.values(), &[1, 2, 3]);-- let name_array = batches[0]- .column(1)- .as_any()- .downcast_ref::<arrow::array::StringArray>()- .expect("Expected StringArray for NAME column");- assert_eq!(name_array.value(0), "a");- assert_eq!(name_array.value(1), "b");- assert_eq!(name_array.value(2), "c");-- Ok(())-}--#[tokio::test]-async fn test_multi_source_schema_adapter_reuse() -> Result<()> {- // This test verifies that the same schema adapter factory can be reused- // across different file source types. This is important for ensuring that:- // 1. The schema adapter factory interface works uniformly across all source types- // 2. The factory can be shared and cloned efficiently using Arc- // 3. Various data source implementations correctly implement the schema adapter factory pattern-- // Create a test factory- let factory = Arc::new(UppercaseAdapterFactory {});-- // Test ArrowFileSource- {- let schema =- Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));- let table_schema = TableSchema::new(schema, vec![]);- let source = ArrowSource::new_file_source(table_schema);- let source_with_adapter = source- .clone()- .with_schema_adapter_factory(factory.clone())- .unwrap();-- let base_source: Arc<dyn FileSource> = source.into();- assert!(base_source.schema_adapter_factory().is_none());- assert!(source_with_adapter.schema_adapter_factory().is_some());-- let retrieved_factory = source_with_adapter.schema_adapter_factory().unwrap();- assert_eq!(- format!("{:?}", retrieved_factory.as_ref()),- format!("{:?}", factory.as_ref())- );- }-- // Test CsvSource- {- let schema =- Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));- let options = CsvOptions {- has_header: Some(true),- delimiter: b',',- quote: b'"',- ..Default::default()- };- let source = CsvSource::new(schema).with_csv_options(options);- let source_with_adapter = source- .clone()- .with_schema_adapter_factory(factory.clone())- .unwrap();-- let base_source: Arc<dyn FileSource> = source.into();- assert!(base_source.schema_adapter_factory().is_none());- assert!(source_with_adapter.schema_adapter_factory().is_some());-- let retrieved_factory = source_with_adapter.schema_adapter_factory().unwrap();- assert_eq!(- format!("{:?}", retrieved_factory.as_ref()),- format!("{:?}", factory.as_ref())- );- }-- // Test JsonSource- {- let schema =- Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));- let table_schema = TableSchema::new(schema, vec![]);- let source = JsonSource::new(table_schema);- let source_with_adapter = source- .clone()- .with_schema_adapter_factory(factory.clone())- .unwrap();-- let base_source: Arc<dyn FileSource> = source.into();- assert!(base_source.schema_adapter_factory().is_none());- assert!(source_with_adapter.schema_adapter_factory().is_some());-- let retrieved_factory = source_with_adapter.schema_adapter_factory().unwrap();- assert_eq!(- format!("{:?}", retrieved_factory.as_ref()),- format!("{:?}", factory.as_ref())- );- }-- Ok(())-}diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rsindex ed0238d788..4c8fd5b340 100644--- a/datafusion/datasource-arrow/src/source.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -34,13 +34,9 @@use std::sync::Arc;use std::{any::Any, io::Cursor};-use datafusion_datasource::schema_adapter::{- DefaultSchemaAdapterFactory, SchemaAdapterFactory,-};use datafusion_datasource::{TableSchema, as_file_source};use arrow::buffer::Buffer;-use arrow::datatypes::SchemaRef;use arrow::ipc::reader::{FileDecoder, FileReader, StreamReader};use datafusion_common::error::Result;use datafusion_common::exec_datafusion_err;@@ -54,7 +50,7 @@ use datafusion_physical_plan::projection::ProjectionExprs;use datafusion_datasource::file_stream::FileOpenFuture;use datafusion_datasource::file_stream::FileOpener;-use futures::{StreamExt, TryStreamExt};+use futures::StreamExt;use itertools::Itertools;use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore};@@ -71,8 +67,6 @@ enum ArrowFormat {pub(crate) struct ArrowStreamFileOpener {object_store: Arc<dyn ObjectStore>,projection: Option<Vec<usize>>,- projected_schema: Option<SchemaRef>,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}impl FileOpener for ArrowStreamFileOpener {@@ -84,8 +78,6 @@ impl FileOpener for ArrowStreamFileOpener {}let object_store = Arc::clone(&self.object_store);let projection = self.projection.clone();- let projected_schema = self.projected_schema.clone();- let schema_adapter_factory = self.schema_adapter_factory.clone();Ok(Box::pin(async move {let r = object_store@@ -111,26 +103,7 @@ impl FileOpener for ArrowStreamFileOpener {}};- // If we have a schema adapter factory and projected schema, use them to normalize the schema- if let (Some(factory), Some(proj_schema)) =- (schema_adapter_factory, projected_schema)- {- Ok(stream- .and_then(move |batch| {- let factory = Arc::clone(&factory);- let proj_schema = Arc::clone(&proj_schema);- async move {- let schema_adapter =- factory.create_with_projected_schema(proj_schema);- let (schema_mapper, _) =- schema_adapter.map_schema(batch.schema().as_ref())?;- schema_mapper.map_batch(batch)- }- })- .boxed())- } else {- Ok(stream)- }+ Ok(stream)}))}}@@ -139,16 +112,12 @@ impl FileOpener for ArrowStreamFileOpener {pub(crate) struct ArrowFileOpener {object_store: Arc<dyn ObjectStore>,projection: Option<Vec<usize>>,- projected_schema: Option<SchemaRef>,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}impl FileOpener for ArrowFileOpener {fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {let object_store = Arc::clone(&self.object_store);let projection = self.projection.clone();- let projected_schema = self.projected_schema.clone();- let schema_adapter_factory = self.schema_adapter_factory.clone();Ok(Box::pin(async move {let range = partitioned_file.range.clone();@@ -176,26 +145,7 @@ impl FileOpener for ArrowFileOpener {}};- // Apply schema adaptation if available- if let (Some(factory), Some(proj_schema)) =- (schema_adapter_factory, projected_schema)- {- Ok(stream- .and_then(move |batch| {- let factory = Arc::clone(&factory);- let proj_schema = Arc::clone(&proj_schema);- async move {- let schema_adapter =- factory.create_with_projected_schema(proj_schema);- let (schema_mapper, _) = schema_adapter- .map_schema(batch.schema().as_ref())?;- schema_mapper.map_batch(batch)- }- })- .boxed())- } else {- Ok(stream)- }+ Ok(stream)}Some(range) => {// range is not none, the file maybe split into multiple parts to scan in parallel@@ -297,27 +247,7 @@ impl FileOpener for ArrowFileOpener {.map(|r| r.map_err(Into::into)).boxed();- // Apply schema adaptation if available- if let (Some(factory), Some(proj_schema)) =- (schema_adapter_factory, projected_schema)- {- Ok(stream- .and_then(move |batch| {- let factory = Arc::clone(&factory);- let proj_schema = Arc::clone(&proj_schema);- async move {- let schema_adapter =- factory.create_with_projected_schema(proj_schema);- let (schema_mapper, projection) = schema_adapter- .map_schema(batch.schema().as_ref())?;- let batch = batch.project(&projection)?;- schema_mapper.map_batch(batch)- }- })- .boxed())- } else {- Ok(stream)- }+ Ok(stream)}}}))@@ -329,7 +259,6 @@ impl FileOpener for ArrowFileOpener {pub struct ArrowSource {format: ArrowFormat,metrics: ExecutionPlanMetricsSet,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,projection: SplitProjection,table_schema: TableSchema,}@@ -341,7 +270,6 @@ impl ArrowSource {Self {format: ArrowFormat::File,metrics: ExecutionPlanMetricsSet::new(),- schema_adapter_factory: None,projection: SplitProjection::unprojected(&table_schema),table_schema,}@@ -353,7 +281,6 @@ impl ArrowSource {Self {format: ArrowFormat::Stream,metrics: ExecutionPlanMetricsSet::new(),- schema_adapter_factory: None,projection: SplitProjection::unprojected(&table_schema),table_schema,}@@ -368,32 +295,15 @@ impl FileSource for ArrowSource {_partition: usize,) -> Result<Arc<dyn FileOpener>> {let split_projection = self.projection.clone();- // For schema adaptation, we only use the file schema (not partition columns)- let projected_file_schema = SchemaRef::from(- self.table_schema- .file_schema()- .project(&split_projection.file_indices)?,- );-- // Use provided schema adapter factory, or default to DefaultSchemaAdapterFactory- // This ensures schema normalization (removing metadata differences) happens during execution- let schema_adapter_factory = self- .schema_adapter_factory- .clone()- .unwrap_or_else(|| Arc::new(DefaultSchemaAdapterFactory));let opener: Arc<dyn FileOpener> = match self.format {ArrowFormat::File => Arc::new(ArrowFileOpener {object_store,projection: Some(split_projection.file_indices.clone()),- projected_schema: Some(Arc::clone(&projected_file_schema)),- schema_adapter_factory: Some(schema_adapter_factory),}),ArrowFormat::Stream => Arc::new(ArrowStreamFileOpener {object_store,projection: Some(split_projection.file_indices.clone()),- projected_schema: Some(projected_file_schema),- schema_adapter_factory: Some(schema_adapter_factory),}),};ProjectionOpener::try_new(@@ -422,20 +332,6 @@ impl FileSource for ArrowSource {}}- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }-fn repartitioned(&self,target_partitions: usize,@@ -527,8 +423,6 @@ impl ArrowOpener {inner: Arc::new(ArrowFileOpener {object_store,projection,- projected_schema: None,- schema_adapter_factory: None,}),}}@@ -541,8 +435,6 @@ impl ArrowOpener {inner: Arc::new(ArrowStreamFileOpener {object_store,projection,- projected_schema: None,- schema_adapter_factory: None,}),}}@@ -738,8 +630,6 @@ mod tests {let opener = ArrowStreamFileOpener {object_store,projection: Some(vec![0]), // just the first column- projected_schema: None,- schema_adapter_factory: None,};let mut stream = opener.open(partitioned_file)?.await?;diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rsindex 33d6cf5272..1c466be266 100644--- a/datafusion/datasource-avro/src/source.rs+++ b/datafusion/datasource-avro/src/source.rs@@ -28,7 +28,6 @@ use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::file_stream::FileOpener;use datafusion_datasource::projection::{ProjectionOpener, SplitProjection};-use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;use datafusion_physical_plan::projection::ProjectionExprs;@@ -42,7 +41,6 @@ pub struct AvroSource {batch_size: Option<usize>,projection: SplitProjection,metrics: ExecutionPlanMetricsSet,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}impl AvroSource {@@ -54,7 +52,6 @@ impl AvroSource {table_schema,batch_size: None,metrics: ExecutionPlanMetricsSet::new(),- schema_adapter_factory: None,}}@@ -142,20 +139,6 @@ impl FileSource for AvroSource {) -> Result<Option<FileScanConfig>> {Ok(None)}-- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }}mod private {diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex 2aed31f1bb..37e3ef23b6 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -18,7 +18,6 @@//! Execution plan for reading CSV filesuse datafusion_datasource::projection::{ProjectionOpener, SplitProjection};-use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_physical_plan::projection::ProjectionExprs;use std::any::Any;use std::fmt;@@ -92,7 +91,6 @@ pub struct CsvSource {table_schema: TableSchema,projection: SplitProjection,metrics: ExecutionPlanMetricsSet,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}impl CsvSource {@@ -105,7 +103,6 @@ impl CsvSource {table_schema,batch_size: None,metrics: ExecutionPlanMetricsSet::new(),- schema_adapter_factory: None,}}@@ -317,20 +314,6 @@ impl FileSource for CsvSource {DisplayFormatType::TreeRender => Ok(()),}}-- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }}impl FileOpener for CsvOpener {diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex 21ffa7f2f9..5797054f11 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -30,7 +30,6 @@ use datafusion_datasource::decoder::{DecoderDeserializer, deserialize_stream};use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};use datafusion_datasource::projection::{ProjectionOpener, SplitProjection};-use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_datasource::{ListingTableUrl, PartitionedFile, RangeCalculation, as_file_source, calculate_range,};@@ -80,7 +79,6 @@ pub struct JsonSource {table_schema: datafusion_datasource::TableSchema,batch_size: Option<usize>,metrics: ExecutionPlanMetricsSet,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,projection: SplitProjection,}@@ -93,7 +91,6 @@ impl JsonSource {table_schema,batch_size: None,metrics: ExecutionPlanMetricsSet::new(),- schema_adapter_factory: None,}}}@@ -172,20 +169,6 @@ impl FileSource for JsonSource {fn file_type(&self) -> &str {"json"}-- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }}impl FileOpener for JsonOpener {diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 4956f83eff..7d7662a4a9 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -264,7 +264,6 @@ use parquet::encryption::decrypt::FileDecryptionProperties;/// filled with nulls, but this can be customized via [`PhysicalExprAdapterFactory`].////// [`RecordBatch`]: arrow::record_batch::RecordBatch-/// [`SchemaAdapter`]: datafusion_datasource::schema_adapter::SchemaAdapter/// [`ParquetMetadata`]: parquet::file::metadata::ParquetMetaData/// [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory#[derive(Clone, Debug)]diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex ce17423455..4aac3d46d4 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -25,6 +25,7 @@ use std::sync::Arc;use crate::file_groups::FileGroupPartitioner;use crate::file_scan_config::FileScanConfig;use crate::file_stream::FileOpener;+#[expect(deprecated)]use crate::schema_adapter::SchemaAdapterFactory;use datafusion_common::config::ConfigOptions;use datafusion_common::{Result, not_impl_err};@@ -191,32 +192,6 @@ pub trait FileSource: Send + Sync {Ok(None)}- /// Set optional schema adapter factory.- ///- /// [`SchemaAdapterFactory`] allows user to specify how fields from the- /// file get mapped to that of the table schema. If you implement this- /// method, you should also implement [`schema_adapter_factory`].- ///- /// The default implementation returns a not implemented error.- ///- /// [`schema_adapter_factory`]: Self::schema_adapter_factory- fn with_schema_adapter_factory(- &self,- _factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- not_impl_err!(- "FileSource {} does not support schema adapter factory",- self.file_type()- )- }-- /// Returns the current schema adapter factory if set- ///- /// Default implementation returns `None`.- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- None- }-/// Set the file ordering information////// This allows the file source to know how the files are sorted,@@ -234,4 +209,35 @@ pub trait FileSource: Send + Sync {// ParquetSource will override thisnot_impl_err!("with_file_ordering_info not implemented for this FileSource")}++ /// Deprecated: Set optional schema adapter factory.+ ///+ /// `SchemaAdapterFactory` has been removed. Use `PhysicalExprAdapterFactory` instead.+ /// See `upgrading.md` for more details.+ #[deprecated(+ since = "52.0.0",+ note = "SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ )]+ #[expect(deprecated)]+ fn with_schema_adapter_factory(+ &self,+ _factory: Arc<dyn SchemaAdapterFactory>,+ ) -> Result<Arc<dyn FileSource>> {+ not_impl_err!(+ "SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ )+ }++ /// Deprecated: Returns the current schema adapter factory if set.+ ///+ /// `SchemaAdapterFactory` has been removed. Use `PhysicalExprAdapterFactory` instead.+ /// See `upgrading.md` for more details.+ #[deprecated(+ since = "52.0.0",+ note = "SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ )]+ #[expect(deprecated)]+ fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {+ None+ }}diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex d4f81df793..b17b5fba68 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -78,7 +78,6 @@ use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::/// # use datafusion_physical_expr::projection::ProjectionExprs;/// # use datafusion_physical_plan::ExecutionPlan;/// # use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;-/// # use datafusion_datasource::schema_adapter::SchemaAdapterFactory;/// # let file_schema = Arc::new(Schema::new(vec![/// # Field::new("c1", DataType::Int32, false),/// # Field::new("c2", DataType::Int32, false),@@ -89,7 +88,6 @@ use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::/// #[derive(Clone)]/// # struct ParquetSource {/// # table_schema: TableSchema,-/// # schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>/// # };/// # impl FileSource for ParquetSource {/// # fn create_file_opener(&self, _: Arc<dyn ObjectStore>, _: &FileScanConfig, _: usize) -> Result<Arc<dyn FileOpener>> { unimplemented!() }@@ -98,13 +96,11 @@ use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::/// # fn with_batch_size(&self, _: usize) -> Arc<dyn FileSource> { unimplemented!() }/// # fn metrics(&self) -> &ExecutionPlanMetricsSet { unimplemented!() }/// # fn file_type(&self) -> &str { "parquet" }-/// # fn with_schema_adapter_factory(&self, factory: Arc<dyn SchemaAdapterFactory>) -> Result<Arc<dyn FileSource>> { Ok(Arc::new(Self {table_schema: self.table_schema.clone(), schema_adapter_factory: Some(factory)} )) }-/// # fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> { self.schema_adapter_factory.clone() }/// # // Note that this implementation drops the projection on the floor, it is not complete!/// # fn try_pushdown_projection(&self, projection: &ProjectionExprs) -> Result<Option<Arc<dyn FileSource>>> { Ok(Some(Arc::new(self.clone()) as Arc<dyn FileSource>)) }/// # }/// # impl ParquetSource {-/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into(), schema_adapter_factory: None} }+/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into()} }/// # }/// // create FileScan config for reading parquet files from file:///// let object_store_url = ObjectStoreUrl::local_filesystem();diff --git a/datafusion/datasource/src/schema_adapter.rs b/datafusion/datasource/src/schema_adapter.rsindex 0ee4ffd648..3d0b06954e 100644--- a/datafusion/datasource/src/schema_adapter.rs+++ b/datafusion/datasource/src/schema_adapter.rs@@ -15,50 +15,47 @@// specific language governing permissions and limitations// under the License.-//! [`SchemaAdapter`] and [`SchemaAdapterFactory`] to adapt file-level record batches to a table schema.+//! Deprecated: [`SchemaAdapter`] and [`SchemaAdapterFactory`] have been removed.//!-//! Adapter provides a method of translating the RecordBatches that come out of the-//! physical format into how they should be used by DataFusion. For instance, a schema-//! can be stored external to a parquet file that maps parquet logical types to arrow types.-use arrow::{- array::{ArrayRef, RecordBatch, RecordBatchOptions, new_null_array},- compute::can_cast_types,- datatypes::{DataType, Field, Schema, SchemaRef},-};-use datafusion_common::{- ColumnStatistics,- format::DEFAULT_CAST_OPTIONS,- nested_struct::{cast_column, validate_struct_compatibility},- plan_err,-};-use std::{fmt::Debug, sync::Arc};-/// Function used by [`SchemaMapping`] to adapt a column from the file schema to-/// the table schema.-pub type CastColumnFn = dyn Fn(- &ArrayRef,- &Field,- &arrow::compute::CastOptions,- ) -> datafusion_common::Result<ArrayRef>+//! Use [`PhysicalExprAdapterFactory`] instead. See `upgrading.md` for more details.+//!+//! [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory++#![allow(deprecated)]++use arrow::array::{ArrayRef, RecordBatch};+use arrow::datatypes::{Field, Schema, SchemaRef};+use datafusion_common::{ColumnStatistics, Result, not_impl_err};+use log::warn;+use std::fmt::Debug;+use std::sync::Arc;++/// Deprecated: Function type for casting columns.+///+/// This type has been removed. Use [`PhysicalExprAdapterFactory`] instead.+/// See `upgrading.md` for more details.+///+/// [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory+#[deprecated(+ since = "52.0.0",+ note = "SchemaAdapter has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+)]+pub type CastColumnFn = dyn Fn(&ArrayRef, &Field, &arrow::compute::CastOptions) -> Result<ArrayRef>+ Send+ Sync;-/// Factory for creating [`SchemaAdapter`]+/// Deprecated: Factory for creating [`SchemaAdapter`].///-/// This interface provides a way to implement custom schema adaptation logic-/// for DataSourceExec (for example, to fill missing columns with default value-/// other than null).+/// This trait has been removed. Use [`PhysicalExprAdapterFactory`] instead.+/// See `upgrading.md` for more details.///-/// Most users should use [`DefaultSchemaAdapterFactory`]. See that struct for-/// more details and examples.+/// [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory+#[deprecated(+ since = "52.0.0",+ note = "SchemaAdapter has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+)]pub trait SchemaAdapterFactory: Debug + Send + Sync + 'static {/// Create a [`SchemaAdapter`]- ///- /// Arguments:- ///- /// * `projected_table_schema`: The schema for the table, projected to- /// include only the fields being output (projected) by the this mapping.- ///- /// * `table_schema`: The entire table schema for the tablefn create(&self,projected_table_schema: SchemaRef,@@ -66,9 +63,6 @@ pub trait SchemaAdapterFactory: Debug + Send + Sync + 'static {) -> Box<dyn SchemaAdapter>;/// Create a [`SchemaAdapter`] using only the projected table schema.- ///- /// This is a convenience method for cases where the table schema and the- /// projected table schema are the same.fn create_with_projected_schema(&self,projected_table_schema: SchemaRef,@@ -77,971 +71,142 @@ pub trait SchemaAdapterFactory: Debug + Send + Sync + 'static {}}-/// Creates [`SchemaMapper`]s to map file-level [`RecordBatch`]es to a table-/// schema, which may have a schema obtained from merging multiple file-level-/// schemas.+/// Deprecated: Creates [`SchemaMapper`]s to map file-level [`RecordBatch`]es to a table schema.///-/// This is useful for implementing schema evolution in partitioned datasets.+/// This trait has been removed. Use [`PhysicalExprAdapterFactory`] instead.+/// See `upgrading.md` for more details.///-/// See [`DefaultSchemaAdapterFactory`] for more details and examples.+/// [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory+#[deprecated(+ since = "52.0.0",+ note = "SchemaAdapter has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+)]pub trait SchemaAdapter: Send + Sync {- /// Map a column index in the table schema to a column index in a particular- /// file schema- ///- /// This is used while reading a file to push down projections by mapping- /// projected column indexes from the table schema to the file schema- ///- /// Panics if index is not in range for the table schema+ /// Map a column index in the table schema to a column index in a particular file schema.fn map_column_index(&self, index: usize, file_schema: &Schema) -> Option<usize>;- /// Creates a mapping for casting columns from the file schema to the table- /// schema.- ///- /// This is used after reading a record batch. The returned [`SchemaMapper`]:- ///- /// 1. Maps columns to the expected columns indexes- /// 2. Handles missing values (e.g. fills nulls or a default value) for- /// columns in the in the table schema not in the file schema- /// 2. Handles different types: if the column in the file schema has a- /// different type than `table_schema`, the mapper will resolve this- /// difference (e.g. by casting to the appropriate type)- ///- /// Returns:- /// * a [`SchemaMapper`]- /// * an ordered list of columns to project from the file+ /// Creates a mapping for casting columns from the file schema to the table schema.fn map_schema(&self,file_schema: &Schema,- ) -> datafusion_common::Result<(Arc<dyn SchemaMapper>, Vec<usize>)>;+ ) -> Result<(Arc<dyn SchemaMapper>, Vec<usize>)>;}-/// Maps, columns from a specific file schema to the table schema.+/// Deprecated: Maps columns from a specific file schema to the table schema.+///+/// This trait has been removed. Use [`PhysicalExprAdapterFactory`] instead.+/// See `upgrading.md` for more details.///-/// See [`DefaultSchemaAdapterFactory`] for more details and examples.+/// [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory+#[deprecated(+ since = "52.0.0",+ note = "SchemaMapper has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+)]pub trait SchemaMapper: Debug + Send + Sync {- /// Adapts a `RecordBatch` to match the `table_schema`- fn map_batch(&self, batch: RecordBatch) -> datafusion_common::Result<RecordBatch>;+ /// Adapts a `RecordBatch` to match the `table_schema`.+ fn map_batch(&self, batch: RecordBatch) -> Result<RecordBatch>;- /// Adapts file-level column `Statistics` to match the `table_schema`+ /// Adapts file-level column `Statistics` to match the `table_schema`.fn map_column_statistics(&self,file_col_statistics: &[ColumnStatistics],- ) -> datafusion_common::Result<Vec<ColumnStatistics>>;+ ) -> Result<Vec<ColumnStatistics>>;}-/// Default [`SchemaAdapterFactory`] for mapping schemas.-///-/// This can be used to adapt file-level record batches to a table schema and-/// implement schema evolution.-///-/// Given an input file schema and a table schema, this factory returns-/// [`SchemaAdapter`] that return [`SchemaMapper`]s that:-///-/// 1. Reorder columns-/// 2. Cast columns to the correct type-/// 3. Fill missing columns with nulls-///-/// # Errors:-///-/// * If a column in the table schema is non-nullable but is not present in the-/// file schema (i.e. it is missing), the returned mapper tries to fill it with-/// nulls resulting in a schema error.-///-/// # Illustration of Schema Mapping-///-/// ```text-/// ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─-/// ┌───────┐ ┌───────┐ │ ┌───────┐ ┌───────┐ ┌───────┐ │-/// ││ 1.0 │ │ "foo" │ ││ NULL │ │ "foo" │ │ "1.0" │-/// ├───────┤ ├───────┤ │ Schema mapping ├───────┤ ├───────┤ ├───────┤ │-/// ││ 2.0 │ │ "bar" │ ││ NULL │ │ "bar" │ │ "2.0" │-/// └───────┘ └───────┘ │────────────────▶ └───────┘ └───────┘ └───────┘ │-/// │ │-/// column "c" column "b"│ column "a" column "b" column "c"│-/// │ Float64 Utf8 │ Int32 Utf8 Utf8-/// ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘-/// Input Record Batch Output Record Batch-///-/// Schema { Schema {-/// "c": Float64, "a": Int32,-/// "b": Utf8, "b": Utf8,-/// } "c": Utf8,-/// }-/// ```-///-/// # Example of using the `DefaultSchemaAdapterFactory` to map [`RecordBatch`]s-///-/// Note `SchemaMapping` also supports mapping partial batches, which is used as-/// part of predicate pushdown.-///-/// ```-/// # use std::sync::Arc;-/// # use arrow::datatypes::{DataType, Field, Schema};-/// # use datafusion_datasource::schema_adapter::{DefaultSchemaAdapterFactory, SchemaAdapterFactory};-/// # use datafusion_common::record_batch;-/// // Table has fields "a", "b" and "c"-/// let table_schema = Schema::new(vec![-/// Field::new("a", DataType::Int32, true),-/// Field::new("b", DataType::Utf8, true),-/// Field::new("c", DataType::Utf8, true),-/// ]);-///-/// // create an adapter to map the table schema to the file schema-/// let adapter = DefaultSchemaAdapterFactory::from_schema(Arc::new(table_schema));+/// Deprecated: Default [`SchemaAdapterFactory`] for mapping schemas.///-/// // The file schema has fields "c" and "b" but "b" is stored as an 'Float64'-/// // instead of 'Utf8'-/// let file_schema = Schema::new(vec![-/// Field::new("c", DataType::Utf8, true),-/// Field::new("b", DataType::Float64, true),-/// ]);+/// This struct has been removed. Use [`PhysicalExprAdapterFactory`] instead.+/// See `upgrading.md` for more details.///-/// // Get a mapping from the file schema to the table schema-/// let (mapper, _indices) = adapter.map_schema(&file_schema).unwrap();-///-/// let file_batch = record_batch!(-/// ("c", Utf8, vec!["foo", "bar"]),-/// ("b", Float64, vec![1.0, 2.0])-/// ).unwrap();-///-/// let mapped_batch = mapper.map_batch(file_batch).unwrap();-///-/// // the mapped batch has the correct schema and the "b" column has been cast to Utf8-/// let expected_batch = record_batch!(-/// ("a", Int32, vec![None, None]), // missing column filled with nulls-/// ("b", Utf8, vec!["1.0", "2.0"]), // b was cast to string and order was changed-/// ("c", Utf8, vec!["foo", "bar"])-/// ).unwrap();-/// assert_eq!(mapped_batch, expected_batch);-/// ```+/// [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory+#[deprecated(+ since = "52.0.0",+ note = "DefaultSchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+)]#[derive(Clone, Debug, Default)]pub struct DefaultSchemaAdapterFactory;-impl DefaultSchemaAdapterFactory {- /// Create a new factory for mapping batches from a file schema to a table- /// schema.- ///- /// This is a convenience for [`DefaultSchemaAdapterFactory::create`] with- /// the same schema for both the projected table schema and the table- /// schema.- pub fn from_schema(table_schema: SchemaRef) -> Box<dyn SchemaAdapter> {- Self.create(Arc::clone(&table_schema), table_schema)- }-}-impl SchemaAdapterFactory for DefaultSchemaAdapterFactory {fn create(&self,projected_table_schema: SchemaRef,_table_schema: SchemaRef,) -> Box<dyn SchemaAdapter> {- Box::new(DefaultSchemaAdapter {- projected_table_schema,+ Box::new(DeprecatedSchemaAdapter {+ _projected_table_schema: projected_table_schema,})}}-/// This SchemaAdapter requires both the table schema and the projected table-/// schema. See [`SchemaMapping`] for more details-#[derive(Clone, Debug)]-pub(crate) struct DefaultSchemaAdapter {- /// The schema for the table, projected to include only the fields being output (projected) by the- /// associated ParquetSource- projected_table_schema: SchemaRef,+impl DefaultSchemaAdapterFactory {+ /// Deprecated: Create a new factory for mapping batches from a file schema to a table schema.+ #[deprecated(+ since = "52.0.0",+ note = "DefaultSchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ )]+ pub fn from_schema(table_schema: SchemaRef) -> Box<dyn SchemaAdapter> {+ // Note: this method did not return an error thus the errors are raised from the returned adapter+ warn!(+ "DefaultSchemaAdapterFactory::from_schema is deprecated. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+ );+ Box::new(DeprecatedSchemaAdapter {+ _projected_table_schema: table_schema,+ })+ }}-/// Checks if a file field can be cast to a table field-///-/// Returns Ok(true) if casting is possible, or an error explaining why casting is not possible-pub(crate) fn can_cast_field(- file_field: &Field,- table_field: &Field,-) -> datafusion_common::Result<bool> {- match (file_field.data_type(), table_field.data_type()) {- (DataType::Struct(source_fields), DataType::Struct(target_fields)) => {- // validate_struct_compatibility returns Result<()>; on success we can cast structs- validate_struct_compatibility(source_fields, target_fields)?;- Ok(true)- }- _ => {- if can_cast_types(file_field.data_type(), table_field.data_type()) {- Ok(true)- } else {- plan_err!(- "Cannot cast file schema field {} of type {} to table schema field of type {}",- file_field.name(),- file_field.data_type(),- table_field.data_type()- )- }- }- }+/// Internal deprecated adapter that returns errors when methods are called.+struct DeprecatedSchemaAdapter {+ _projected_table_schema: SchemaRef,}-impl SchemaAdapter for DefaultSchemaAdapter {- /// Map a column index in the table schema to a column index in a particular- /// file schema- ///- /// Panics if index is not in range for the table schema- fn map_column_index(&self, index: usize, file_schema: &Schema) -> Option<usize> {- let field = self.projected_table_schema.field(index);- Some(file_schema.fields.find(field.name())?.0)+impl SchemaAdapter for DeprecatedSchemaAdapter {+ fn map_column_index(&self, _index: usize, _file_schema: &Schema) -> Option<usize> {+ None // Safe no-op}- /// Creates a `SchemaMapping` for casting or mapping the columns from the- /// file schema to the table schema.- ///- /// If the provided `file_schema` contains columns of a different type to- /// the expected `table_schema`, the method will attempt to cast the array- /// data from the file schema to the table schema where possible.- ///- /// Returns a [`SchemaMapping`] that can be applied to the output batch- /// along with an ordered list of columns to project from the filefn map_schema(&self,- file_schema: &Schema,- ) -> datafusion_common::Result<(Arc<dyn SchemaMapper>, Vec<usize>)> {- let (field_mappings, projection) = create_field_mapping(- file_schema,- &self.projected_table_schema,- can_cast_field,- )?;-- Ok((- Arc::new(SchemaMapping::new(- Arc::clone(&self.projected_table_schema),- field_mappings,- Arc::new(- |array: &ArrayRef,- field: &Field,- opts: &arrow::compute::CastOptions| {- cast_column(array, field, opts)- },- ),- )),- projection,- ))- }-}--/// Helper function that creates field mappings between file schema and table schema-///-/// Maps columns from the file schema to their corresponding positions in the table schema,-/// applying type compatibility checking via the provided predicate function.-///-/// Returns field mappings (for column reordering) and a projection (for field selection).-pub(crate) fn create_field_mapping<F>(- file_schema: &Schema,- projected_table_schema: &SchemaRef,- can_map_field: F,-) -> datafusion_common::Result<(Vec<Option<usize>>, Vec<usize>)>-where- F: Fn(&Field, &Field) -> datafusion_common::Result<bool>,-{- let mut projection = Vec::with_capacity(file_schema.fields().len());- let mut field_mappings = vec![None; projected_table_schema.fields().len()];-- for (file_idx, file_field) in file_schema.fields.iter().enumerate() {- if let Some((table_idx, table_field)) =- projected_table_schema.fields().find(file_field.name())- && can_map_field(file_field, table_field)?- {- field_mappings[table_idx] = Some(projection.len());- projection.push(file_idx);- }+ _file_schema: &Schema,+ ) -> Result<(Arc<dyn SchemaMapper>, Vec<usize>)> {+ not_impl_err!(+ "SchemaAdapter has been removed. Use PhysicalExprAdapterFactory instead. \+ See upgrading.md for more details."+ )}-- Ok((field_mappings, projection))}-/// The SchemaMapping struct holds a mapping from the file schema to the table-/// schema and any necessary type conversions.+/// Deprecated: The SchemaMapping struct held a mapping from the file schema to the table schema.///-/// [`map_batch`] is used by the ParquetOpener to produce a RecordBatch which-/// has the projected schema, since that's the schema which is supposed to come-/// out of the execution of this query. Thus `map_batch` uses-/// `projected_table_schema` as it can only operate on the projected fields.+/// This struct has been removed. Use [`PhysicalExprAdapterFactory`] instead.+/// See `upgrading.md` for more details.///-/// [`map_batch`]: Self::map_batch+/// [`PhysicalExprAdapterFactory`]: datafusion_physical_expr_adapter::PhysicalExprAdapterFactory+#[deprecated(+ since = "52.0.0",+ note = "SchemaMapping has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details."+)]+#[derive(Debug)]pub struct SchemaMapping {- /// The schema of the table. This is the expected schema after conversion- /// and it should match the schema of the query result.- projected_table_schema: SchemaRef,- /// Mapping from field index in `projected_table_schema` to index in- /// projected file_schema.- ///- /// They are Options instead of just plain `usize`s because the table could- /// have fields that don't exist in the file.- field_mappings: Vec<Option<usize>>,- /// Function used to adapt a column from the file schema to the table schema- /// when it exists in both schemas- cast_column: Arc<CastColumnFn>,-}--impl Debug for SchemaMapping {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_struct("SchemaMapping")- .field("projected_table_schema", &self.projected_table_schema)- .field("field_mappings", &self.field_mappings)- .field("cast_column", &"<fn>")- .finish()- }-}--impl SchemaMapping {- /// Creates a new SchemaMapping instance- ///- /// Initializes the field mappings needed to transform file data to the projected table schema- pub fn new(- projected_table_schema: SchemaRef,- field_mappings: Vec<Option<usize>>,- cast_column: Arc<CastColumnFn>,- ) -> Self {- Self {- projected_table_schema,- field_mappings,- cast_column,- }- }+ // Private fields removed - this is a skeleton for deprecation purposes only+ _private: (),}impl SchemaMapper for SchemaMapping {- /// Adapts a `RecordBatch` to match the `projected_table_schema` using the stored mapping and- /// conversions.- /// The produced RecordBatch has a schema that contains only the projected columns.- fn map_batch(&self, batch: RecordBatch) -> datafusion_common::Result<RecordBatch> {- let (_old_schema, batch_cols, batch_rows) = batch.into_parts();-- let cols = self- .projected_table_schema- // go through each field in the projected schema- .fields()- .iter()- // and zip it with the index that maps fields from the projected table schema to the- // projected file schema in `batch`- .zip(&self.field_mappings)- // and for each one...- .map(|(field, file_idx)| {- file_idx.map_or_else(- // If this field only exists in the table, and not in the file, then we know- // that it's null, so just return that.- || Ok(new_null_array(field.data_type(), batch_rows)),- // However, if it does exist in both, use the cast_column function- // to perform any necessary conversions- |batch_idx| {- (self.cast_column)(- &batch_cols[batch_idx],- field,- &DEFAULT_CAST_OPTIONS,- )- },- )- })- .collect::<datafusion_common::Result<Vec<_>, _>>()?;-- // Necessary to handle empty batches- let options = RecordBatchOptions::new().with_row_count(Some(batch_rows));-- let schema = Arc::clone(&self.projected_table_schema);- let record_batch = RecordBatch::try_new_with_options(schema, cols, &options)?;- Ok(record_batch)+ fn map_batch(&self, _batch: RecordBatch) -> Result<RecordBatch> {+ not_impl_err!(+ "SchemaMapping has been removed. Use PhysicalExprAdapterFactory instead. \+ See upgrading.md for more details."+ )}- /// Adapts file-level column `Statistics` to match the `table_schema`fn map_column_statistics(&self,- file_col_statistics: &[ColumnStatistics],- ) -> datafusion_common::Result<Vec<ColumnStatistics>> {- let mut table_col_statistics = vec![];-- // Map the statistics for each field in the file schema to the corresponding field in the- // table schema, if a field is not present in the file schema, we need to fill it with `ColumnStatistics::new_unknown`- for (_, file_col_idx) in self- .projected_table_schema- .fields()- .iter()- .zip(&self.field_mappings)- {- if let Some(file_col_idx) = file_col_idx {- table_col_statistics.push(- file_col_statistics- .get(*file_col_idx)- .cloned()- .unwrap_or_default(),- );- } else {- table_col_statistics.push(ColumnStatistics::new_unknown());- }- }-- Ok(table_col_statistics)- }-}--#[cfg(test)]-mod tests {- use super::*;- use arrow::{- array::{Array, ArrayRef, StringBuilder, StructArray, TimestampMillisecondArray},- compute::cast,- datatypes::{DataType, Field, TimeUnit},- record_batch::RecordBatch,- };- use datafusion_common::{Result, ScalarValue, Statistics, stats::Precision};-- #[test]- fn test_schema_mapping_map_statistics_basic() {- // Create table schema (a, b, c)- let table_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- Field::new("c", DataType::Float64, true),- ]));-- // Create file schema (b, a) - different order, missing c- let file_schema = Schema::new(vec![- Field::new("b", DataType::Utf8, true),- Field::new("a", DataType::Int32, true),- ]);-- // Create SchemaAdapter- let adapter = DefaultSchemaAdapter {- projected_table_schema: Arc::clone(&table_schema),- };-- // Get mapper and projection- let (mapper, projection) = adapter.map_schema(&file_schema).unwrap();-- // Should project columns 0,1 from file- assert_eq!(projection, vec![0, 1]);-- // Create file statistics- let mut file_stats = Statistics::default();-- // Statistics for column b (index 0 in file)- let b_stats = ColumnStatistics {- null_count: Precision::Exact(5),- ..Default::default()- };-- // Statistics for column a (index 1 in file)- let a_stats = ColumnStatistics {- null_count: Precision::Exact(10),- ..Default::default()- };-- file_stats.column_statistics = vec![b_stats, a_stats];-- // Map statistics- let table_col_stats = mapper- .map_column_statistics(&file_stats.column_statistics)- .unwrap();-- // Verify stats- assert_eq!(table_col_stats.len(), 3);- assert_eq!(table_col_stats[0].null_count, Precision::Exact(10)); // a from file idx 1- assert_eq!(table_col_stats[1].null_count, Precision::Exact(5)); // b from file idx 0- assert_eq!(table_col_stats[2].null_count, Precision::Absent); // c (unknown)- }-- #[test]- fn test_schema_mapping_map_statistics_empty() {- // Create schemas- let table_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- ]));- let file_schema = Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- ]);-- let adapter = DefaultSchemaAdapter {- projected_table_schema: Arc::clone(&table_schema),- };- let (mapper, _) = adapter.map_schema(&file_schema).unwrap();-- // Empty file statistics- let file_stats = Statistics::default();- let table_col_stats = mapper- .map_column_statistics(&file_stats.column_statistics)- .unwrap();-- // All stats should be unknown- assert_eq!(table_col_stats.len(), 2);- assert_eq!(table_col_stats[0], ColumnStatistics::new_unknown(),);- assert_eq!(table_col_stats[1], ColumnStatistics::new_unknown(),);- }-- #[test]- fn test_can_cast_field() {- // Same type should work- let from_field = Field::new("col", DataType::Int32, true);- let to_field = Field::new("col", DataType::Int32, true);- assert!(can_cast_field(&from_field, &to_field).unwrap());-- // Casting Int32 to Float64 is allowed- let from_field = Field::new("col", DataType::Int32, true);- let to_field = Field::new("col", DataType::Float64, true);- assert!(can_cast_field(&from_field, &to_field).unwrap());-- // Casting Float64 to Utf8 should work (converts to string)- let from_field = Field::new("col", DataType::Float64, true);- let to_field = Field::new("col", DataType::Utf8, true);- assert!(can_cast_field(&from_field, &to_field).unwrap());-- // Binary to Utf8 is not supported - this is an example of a cast that should fail- // Note: We use Binary instead of Utf8->Int32 because Arrow actually supports that cast- let from_field = Field::new("col", DataType::Binary, true);- let to_field = Field::new("col", DataType::Decimal128(10, 2), true);- let result = can_cast_field(&from_field, &to_field);- assert!(result.is_err());- let error_msg = result.unwrap_err().to_string();- assert!(error_msg.contains("Cannot cast file schema field col"));- }-- #[test]- fn test_create_field_mapping() {- // Define the table schema- let table_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- Field::new("c", DataType::Float64, true),- ]));-- // Define file schema: different order, missing column c, and b has different type- let file_schema = Schema::new(vec![- Field::new("b", DataType::Float64, true), // Different type but castable to Utf8- Field::new("a", DataType::Int32, true), // Same type- Field::new("d", DataType::Boolean, true), // Not in table schema- ]);-- // Custom can_map_field function that allows all mappings for testing- let allow_all = |_: &Field, _: &Field| Ok(true);-- // Test field mapping- let (field_mappings, projection) =- create_field_mapping(&file_schema, &table_schema, allow_all).unwrap();-- // Expected:- // - field_mappings[0] (a) maps to projection[1]- // - field_mappings[1] (b) maps to projection[0]- // - field_mappings[2] (c) is None (not in file)- assert_eq!(field_mappings, vec![Some(1), Some(0), None]);- assert_eq!(projection, vec![0, 1]); // Projecting file columns b, a-- // Test with a failing mapper- let fails_all = |_: &Field, _: &Field| Ok(false);- let (field_mappings, projection) =- create_field_mapping(&file_schema, &table_schema, fails_all).unwrap();-- // Should have no mappings or projections if all cast checks fail- assert_eq!(field_mappings, vec![None, None, None]);- assert_eq!(projection, Vec::<usize>::new());-- // Test with error-producing mapper- let error_mapper = |_: &Field, _: &Field| plan_err!("Test error");- let result = create_field_mapping(&file_schema, &table_schema, error_mapper);- assert!(result.is_err());- assert!(result.unwrap_err().to_string().contains("Test error"));- }-- #[test]- fn test_schema_mapping_new() {- // Define the projected table schema- let projected_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- ]));-- // Define field mappings from table to file- let field_mappings = vec![Some(1), Some(0)];-- // Create SchemaMapping manually- let mapping = SchemaMapping::new(- Arc::clone(&projected_schema),- field_mappings.clone(),- Arc::new(- |array: &ArrayRef, field: &Field, opts: &arrow::compute::CastOptions| {- cast_column(array, field, opts)- },- ),- );-- // Check that fields were set correctly- assert_eq!(*mapping.projected_table_schema, *projected_schema);- assert_eq!(mapping.field_mappings, field_mappings);-- // Test with a batch to ensure it works properly- let batch = RecordBatch::try_new(- Arc::new(Schema::new(vec![- Field::new("b_file", DataType::Utf8, true),- Field::new("a_file", DataType::Int32, true),- ])),- vec![- Arc::new(arrow::array::StringArray::from(vec!["hello", "world"])),- Arc::new(arrow::array::Int32Array::from(vec![1, 2])),- ],- )- .unwrap();-- // Test that map_batch works with our manually created mapping- let mapped_batch = mapping.map_batch(batch).unwrap();-- // Verify the mapped batch has the correct schema and data- assert_eq!(*mapped_batch.schema(), *projected_schema);- assert_eq!(mapped_batch.num_columns(), 2);- assert_eq!(mapped_batch.column(0).len(), 2); // a column- assert_eq!(mapped_batch.column(1).len(), 2); // b column- }-- #[test]- fn test_map_schema_error_path() {- // Define the table schema- let table_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- Field::new("c", DataType::Decimal128(10, 2), true), // Use Decimal which has stricter cast rules- ]));-- // Define file schema with incompatible type for column c- let file_schema = Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Float64, true), // Different but castable- Field::new("c", DataType::Binary, true), // Not castable to Decimal128- ]);-- // Create DefaultSchemaAdapter- let adapter = DefaultSchemaAdapter {- projected_table_schema: Arc::clone(&table_schema),- };-- // map_schema should error due to incompatible types- let result = adapter.map_schema(&file_schema);- assert!(result.is_err());- let error_msg = result.unwrap_err().to_string();- assert!(error_msg.contains("Cannot cast file schema field c"));- }-- #[test]- fn test_map_schema_happy_path() {- // Define the table schema- let table_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Utf8, true),- Field::new("c", DataType::Decimal128(10, 2), true),- ]));-- // Create DefaultSchemaAdapter- let adapter = DefaultSchemaAdapter {- projected_table_schema: Arc::clone(&table_schema),- };-- // Define compatible file schema (missing column c)- let compatible_file_schema = Schema::new(vec![- Field::new("a", DataType::Int64, true), // Can be cast to Int32- Field::new("b", DataType::Float64, true), // Can be cast to Utf8- ]);-- // Test successful schema mapping- let (mapper, projection) = adapter.map_schema(&compatible_file_schema).unwrap();-- // Verify field_mappings and projection created correctly- assert_eq!(projection, vec![0, 1]); // Projecting a and b-- // Verify the SchemaMapping works with actual data- let file_batch = RecordBatch::try_new(- Arc::new(compatible_file_schema.clone()),- vec![- Arc::new(arrow::array::Int64Array::from(vec![100, 200])),- Arc::new(arrow::array::Float64Array::from(vec![1.5, 2.5])),- ],+ _file_col_statistics: &[ColumnStatistics],+ ) -> Result<Vec<ColumnStatistics>> {+ not_impl_err!(+ "SchemaMapping has been removed. Use PhysicalExprAdapterFactory instead. \+ See upgrading.md for more details.")- .unwrap();-- let mapped_batch = mapper.map_batch(file_batch).unwrap();-- // Verify correct schema mapping- assert_eq!(*mapped_batch.schema(), *table_schema);- assert_eq!(mapped_batch.num_columns(), 3); // a, b, c-- // Column c should be null since it wasn't in the file schema- let c_array = mapped_batch.column(2);- assert_eq!(c_array.len(), 2);- assert_eq!(c_array.null_count(), 2);- }-- #[test]- fn test_adapt_struct_with_added_nested_fields() -> Result<()> {- let (file_schema, table_schema) = create_test_schemas_with_nested_fields();- let batch = create_test_batch_with_struct_data(&file_schema)?;-- let adapter = DefaultSchemaAdapter {- projected_table_schema: Arc::clone(&table_schema),- };- let (mapper, _) = adapter.map_schema(file_schema.as_ref())?;- let mapped_batch = mapper.map_batch(batch)?;-- verify_adapted_batch_with_nested_fields(&mapped_batch, &table_schema)?;- Ok(())- }-- #[test]- fn test_map_column_statistics_struct() -> Result<()> {- let (file_schema, table_schema) = create_test_schemas_with_nested_fields();-- let adapter = DefaultSchemaAdapter {- projected_table_schema: Arc::clone(&table_schema),- };- let (mapper, _) = adapter.map_schema(file_schema.as_ref())?;-- let file_stats = vec![- create_test_column_statistics(- 0,- 100,- Some(ScalarValue::Int32(Some(1))),- Some(ScalarValue::Int32(Some(100))),- Some(ScalarValue::Int32(Some(5100))),- ),- create_test_column_statistics(10, 50, None, None, None),- ];-- let table_stats = mapper.map_column_statistics(&file_stats)?;- assert_eq!(table_stats.len(), 1);- verify_column_statistics(- &table_stats[0],- Some(0),- Some(100),- Some(ScalarValue::Int32(Some(1))),- Some(ScalarValue::Int32(Some(100))),- Some(ScalarValue::Int32(Some(5100))),- );- let missing_stats = mapper.map_column_statistics(&[])?;- assert_eq!(missing_stats.len(), 1);- assert_eq!(missing_stats[0], ColumnStatistics::new_unknown());- Ok(())- }-- fn create_test_schemas_with_nested_fields() -> (SchemaRef, SchemaRef) {- let file_schema = Arc::new(Schema::new(vec![Field::new(- "info",- DataType::Struct(- vec![- Field::new("location", DataType::Utf8, true),- Field::new(- "timestamp_utc",- DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),- true,- ),- ]- .into(),- ),- true,- )]));-- let table_schema = Arc::new(Schema::new(vec![Field::new(- "info",- DataType::Struct(- vec![- Field::new("location", DataType::Utf8, true),- Field::new(- "timestamp_utc",- DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),- true,- ),- Field::new(- "reason",- DataType::Struct(- vec![- Field::new("_level", DataType::Float64, true),- Field::new(- "details",- DataType::Struct(- vec![- Field::new("rurl", DataType::Utf8, true),- Field::new("s", DataType::Float64, true),- Field::new("t", DataType::Utf8, true),- ]- .into(),- ),- true,- ),- ]- .into(),- ),- true,- ),- ]- .into(),- ),- true,- )]));-- (file_schema, table_schema)- }-- fn create_test_batch_with_struct_data(- file_schema: &SchemaRef,- ) -> Result<RecordBatch> {- let mut location_builder = StringBuilder::new();- location_builder.append_value("San Francisco");- location_builder.append_value("New York");-- let timestamp_array = TimestampMillisecondArray::from(vec![- Some(1640995200000),- Some(1641081600000),- ]);-- let timestamp_type =- DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into()));- let timestamp_array = cast(×tamp_array, ×tamp_type)?;-- let info_struct = StructArray::from(vec![- (- Arc::new(Field::new("location", DataType::Utf8, true)),- Arc::new(location_builder.finish()) as ArrayRef,- ),- (- Arc::new(Field::new("timestamp_utc", timestamp_type, true)),- timestamp_array,- ),- ]);-- Ok(RecordBatch::try_new(- Arc::clone(file_schema),- vec![Arc::new(info_struct)],- )?)- }-- fn verify_adapted_batch_with_nested_fields(- mapped_batch: &RecordBatch,- table_schema: &SchemaRef,- ) -> Result<()> {- assert_eq!(mapped_batch.schema(), *table_schema);- assert_eq!(mapped_batch.num_rows(), 2);-- let info_col = mapped_batch.column(0);- let info_array = info_col- .as_any()- .downcast_ref::<StructArray>()- .expect("Expected info column to be a StructArray");-- verify_preserved_fields(info_array)?;- verify_reason_field_structure(info_array)?;- Ok(())- }-- fn verify_preserved_fields(info_array: &StructArray) -> Result<()> {- let location_col = info_array- .column_by_name("location")- .expect("Expected location field in struct");- let location_array = location_col- .as_any()- .downcast_ref::<arrow::array::StringArray>()- .expect("Expected location to be a StringArray");- assert_eq!(location_array.value(0), "San Francisco");- assert_eq!(location_array.value(1), "New York");-- let timestamp_col = info_array- .column_by_name("timestamp_utc")- .expect("Expected timestamp_utc field in struct");- let timestamp_array = timestamp_col- .as_any()- .downcast_ref::<TimestampMillisecondArray>()- .expect("Expected timestamp_utc to be a TimestampMillisecondArray");- assert_eq!(timestamp_array.value(0), 1640995200000);- assert_eq!(timestamp_array.value(1), 1641081600000);- Ok(())- }-- fn verify_reason_field_structure(info_array: &StructArray) -> Result<()> {- let reason_col = info_array- .column_by_name("reason")- .expect("Expected reason field in struct");- let reason_array = reason_col- .as_any()- .downcast_ref::<StructArray>()- .expect("Expected reason to be a StructArray");- assert_eq!(reason_array.fields().len(), 2);- assert!(reason_array.column_by_name("_level").is_some());- assert!(reason_array.column_by_name("details").is_some());-- let details_col = reason_array- .column_by_name("details")- .expect("Expected details field in reason struct");- let details_array = details_col- .as_any()- .downcast_ref::<StructArray>()- .expect("Expected details to be a StructArray");- assert_eq!(details_array.fields().len(), 3);- assert!(details_array.column_by_name("rurl").is_some());- assert!(details_array.column_by_name("s").is_some());- assert!(details_array.column_by_name("t").is_some());- for i in 0..2 {- assert!(reason_array.is_null(i), "reason field should be null");- }- Ok(())- }-- fn verify_column_statistics(- stats: &ColumnStatistics,- expected_null_count: Option<usize>,- expected_distinct_count: Option<usize>,- expected_min: Option<ScalarValue>,- expected_max: Option<ScalarValue>,- expected_sum: Option<ScalarValue>,- ) {- if let Some(count) = expected_null_count {- assert_eq!(- stats.null_count,- Precision::Exact(count),- "Null count should match expected value"- );- }- if let Some(count) = expected_distinct_count {- assert_eq!(- stats.distinct_count,- Precision::Exact(count),- "Distinct count should match expected value"- );- }- if let Some(min) = expected_min {- assert_eq!(- stats.min_value,- Precision::Exact(min),- "Min value should match expected value"- );- }- if let Some(max) = expected_max {- assert_eq!(- stats.max_value,- Precision::Exact(max),- "Max value should match expected value"- );- }- if let Some(sum) = expected_sum {- assert_eq!(- stats.sum_value,- Precision::Exact(sum),- "Sum value should match expected value"- );- }- }-- fn create_test_column_statistics(- null_count: usize,- distinct_count: usize,- min_value: Option<ScalarValue>,- max_value: Option<ScalarValue>,- sum_value: Option<ScalarValue>,- ) -> ColumnStatistics {- ColumnStatistics {- null_count: Precision::Exact(null_count),- distinct_count: Precision::Exact(distinct_count),- min_value: min_value.map_or_else(|| Precision::Absent, Precision::Exact),- max_value: max_value.map_or_else(|| Precision::Absent, Precision::Exact),- sum_value: sum_value.map_or_else(|| Precision::Absent, Precision::Exact),- byte_size: Precision::Absent,- }}}diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rsindex a7eb7fd1c4..c8d5dd54cb 100644--- a/datafusion/datasource/src/test_util.rs+++ b/datafusion/datasource/src/test_util.rs@@ -17,7 +17,6 @@use crate::{file::FileSource, file_scan_config::FileScanConfig, file_stream::FileOpener,- schema_adapter::SchemaAdapterFactory,};use std::sync::Arc;@@ -32,7 +31,6 @@ use object_store::ObjectStore;#[derive(Clone)]pub(crate) struct MockSource {metrics: ExecutionPlanMetricsSet,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,filter: Option<Arc<dyn PhysicalExpr>>,table_schema: crate::table_schema::TableSchema,projection: crate::projection::SplitProjection,@@ -44,7 +42,6 @@ impl Default for MockSource {crate::table_schema::TableSchema::new(Arc::new(Schema::empty()), vec![]);Self {metrics: ExecutionPlanMetricsSet::new(),- schema_adapter_factory: None,filter: None,projection: crate::projection::SplitProjection::unprojected(&table_schema),table_schema,@@ -57,7 +54,6 @@ impl MockSource {let table_schema = table_schema.into();Self {metrics: ExecutionPlanMetricsSet::new(),- schema_adapter_factory: None,filter: None,projection: crate::projection::SplitProjection::unprojected(&table_schema),table_schema,@@ -100,20 +96,6 @@ impl FileSource for MockSource {"mock"}- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }-fn table_schema(&self) -> &crate::table_schema::TableSchema {&self.table_schema}diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rsindex e9b8ff5c37..83727ac092 100644--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs@@ -408,8 +408,8 @@ impl<'a> DefaultPhysicalExprAdapterRewriter<'a> {column.name());}- // If the column is missing from the physical schema fill it in with nulls as `SchemaAdapter` used to do.- // If users want a different behavior they need to provide a custom `PhysicalExprAdapter` implementation.+ // If the column is missing from the physical schema fill it in with nulls.+ // For a different behavior, provide a custom `PhysicalExprAdapter` implementation.let null_value = ScalarValue::Null.cast_to(logical_field.data_type())?;return Ok(Transformed::yes(expressions::lit(null_value)));}diff --git a/datafusion/sqllogictest/test_files/schema_evolution.slt b/datafusion/sqllogictest/test_files/schema_evolution.sltindex 5572c4a5ff..e29aa14f13 100644--- a/datafusion/sqllogictest/test_files/schema_evolution.slt+++ b/datafusion/sqllogictest/test_files/schema_evolution.slt@@ -138,3 +138,147 @@ select * from parquet_table where c > 11.0;----bzz 300 13.7foo 200 12.6++##########+# Projection tests - selecting subset of columns+# These tests verify column reordering and projection work correctly+# with schema evolution (addresses E2E column reordering concern)+##########++# Select only column a+query T rowsort+select a from parquet_table;+----+NULL+bzz+foo+foo+foo+foo+foo+foo+foo++# Select columns in different order than table schema (c, a instead of a, b, c)+query RT rowsort+select c, a from parquet_table;+----+10.5 foo+12.6 foo+13.7 bzz+NULL NULL+NULL foo+NULL foo+NULL foo+NULL foo+NULL foo++# Select single column that's missing in some files+query I rowsort+select b from parquet_table;+----+1+10+100+2+200+3+300+NULL+NULL++##########+# Projection with filter tests+##########++# Projection with equality filter+query TI rowsort+select a, b from parquet_table where a = 'foo';+----+foo 1+foo 100+foo 2+foo 200+foo 3+foo NULL+foo NULL++# Projection with range filter on projected column+query IR rowsort+select b, c from parquet_table where b > 5;+----+10 NULL+100 10.5+200 12.6+300 13.7++# Projection excluding filtered column (filter on c, project a)+query T rowsort+select a from parquet_table where c > 11.0;+----+bzz+foo++##########+# Complex filter tests - OR combinations and IS NOT NULL+##########++# OR combination+query TIR rowsort+select * from parquet_table where a = 'foo' OR b > 100;+----+bzz 300 13.7+foo 1 NULL+foo 100 10.5+foo 2 NULL+foo 200 12.6+foo 3 NULL+foo NULL NULL+foo NULL NULL++# IS NOT NULL on column a+query TIR rowsort+select * from parquet_table where a IS NOT NULL;+----+bzz 300 13.7+foo 1 NULL+foo 100 10.5+foo 2 NULL+foo 200 12.6+foo 3 NULL+foo NULL NULL+foo NULL NULL++# IS NOT NULL on column c (missing in most files)+query TIR rowsort+select * from parquet_table where c IS NOT NULL;+----+bzz 300 13.7+foo 100 10.5+foo 200 12.6++# Combined conditions with NULL checks+query TIR rowsort+select * from parquet_table where a IS NULL OR (b IS NOT NULL AND b > 5);+----+NULL 10 NULL+bzz 300 13.7+foo 100 10.5+foo 200 12.6++##########+# Multi-column predicates+##########++# AND across columns with different availability+query TIR rowsort+select * from parquet_table where a = 'foo' AND b > 50;+----+foo 100 10.5+foo 200 12.6++# Filter on multiple columns from reordered file (File4 has b, a, c order)+query TIR rowsort+select * from parquet_table where b = 100 AND c = 10.5;+----+foo 100 10.5diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex bd9fb3f55a..c12768c95a 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -513,17 +513,28 @@ let config = FileScanConfigBuilder::new(url, source).build();```-### `SchemaAdapterFactory` Fully Removed from Parquet+### `SchemaAdapter` and `SchemaAdapterFactory` completely removedFollowing the deprecation announced in [DataFusion 49.0.0](#deprecating-schemaadapterfactory-and-schemaadapter), `SchemaAdapterFactory` has been fully removed from Parquet scanning. This applies to both:-- **Predicate pushdown / row filtering** (deprecated in 49.0.0)-- **Projections** (newly removed in 52.0.0)+The following symbols have been deprecated and will be removed in the next release:-If you were using a custom `SchemaAdapterFactory` for schema adaptation (e.g., default column values, type coercion), you should now implement `PhysicalExprAdapterFactory` instead.+- `SchemaAdapter` trait+- `SchemaAdapterFactory` trait+- `SchemaMapper` trait+- `SchemaMapping` struct+- `DefaultSchemaAdapterFactory` struct+These types were previously used to adapt record batch schemas during file reading.+This functionality has been replaced by `PhysicalExprAdapterFactory`, which rewrites expressions at planning time rather than transforming batches at runtime.+If you were using a custom `SchemaAdapterFactory` for schema adaptation (e.g., default column values, type coercion), you should now implement `PhysicalExprAdapterFactory` instead.See the [default column values example](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_data_source/default_column_values.rs) for how to implement a custom `PhysicalExprAdapterFactory`.+**Migration guide:**++If you implemented a custom `SchemaAdapterFactory`, migrate to `PhysicalExprAdapterFactory`.+See the [default column values example](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_data_source/default_column_values.rs) for a complete implementation.+## DataFusion `51.0.0`### `arrow` / `parquet` updated to 57.0.0