Apache DataFusion SQL Query Engine
feat: allow downcasting `ParquetFileReaderFactory` (#26635)
Morris9f8982181ec7
diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rsindex 1452bc8ef8..f9a5e11fdd 100644--- a/datafusion/datasource-parquet/src/reader.rs+++ b/datafusion/datasource-parquet/src/reader.rs@@ -55,7 +55,7 @@ pub trait ParquetFileSchemaProvider: Debug + Send + Sync + 'static {/// such as pre-cached metadata, I/O coalescing, etc.////// See [`DefaultParquetFileReaderFactory`] for a simple implementation.-pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {+pub trait ParquetFileReaderFactory: Any + Debug + Send + Sync {/// Provides an `AsyncFileReader` for reading data from a parquet file specified////// # Notes@@ -79,6 +79,18 @@ pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>>;}+impl dyn ParquetFileReaderFactory {+ /// Returns `true` if this factory is of type `T`.+ pub fn is<T: ParquetFileReaderFactory>(&self) -> bool {+ (self as &dyn Any).is::<T>()+ }++ /// Attempts to downcast this factory to a concrete type `T`.+ pub fn downcast_ref<T: ParquetFileReaderFactory>(&self) -> Option<&T> {+ (self as &dyn Any).downcast_ref()+ }+}+/// Default implementation of [`ParquetFileReaderFactory`]////// This implementation:diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex c818cb3512..09388b0963 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -1518,6 +1518,107 @@ mod tests {assert!(source.filter().is_some());}+ #[test]+ fn parquet_file_reader_factory_downcasts_to_concrete_type() {+ use crate::CachedParquetFileReaderFactory;+ use datafusion_datasource::PartitionedFile;+ use datafusion_execution::cache::cache_manager::FileMetadataCache;+ use datafusion_execution::cache::default_cache::DefaultCache;+ use object_store::memory::InMemory;+ use parquet::arrow::async_reader::AsyncFileReader;++ let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());+ let source = ParquetSource::new(Arc::new(Schema::empty()))+ .with_parquet_file_reader_factory(Arc::new(+ DefaultParquetFileReaderFactory::new(Arc::clone(&store)),+ ));+ let factory = source.parquet_file_reader_factory().unwrap();+ assert!(factory.is::<DefaultParquetFileReaderFactory>());+ assert!(+ factory+ .downcast_ref::<DefaultParquetFileReaderFactory>()+ .is_some()+ );+ assert!(+ factory+ .downcast_ref::<CachedParquetFileReaderFactory>()+ .is_none()+ );++ let cache: Arc<FileMetadataCache> = Arc::new(DefaultCache::new(1024));+ let source = ParquetSource::new(Arc::new(Schema::empty()))+ .with_parquet_file_reader_factory(Arc::new(+ CachedParquetFileReaderFactory::new(Arc::clone(&store), cache),+ ));+ let factory = source.parquet_file_reader_factory().unwrap();+ assert!(factory.is::<CachedParquetFileReaderFactory>());+ assert!(+ factory+ .downcast_ref::<CachedParquetFileReaderFactory>()+ .is_some()+ );+ assert!(+ factory+ .downcast_ref::<DefaultParquetFileReaderFactory>()+ .is_none()+ );++ // A custom factory whose Debug output resembles a built-in factory.+ // Verifies that concrete-type downcasting avoids false-positive matches+ // from Debug substring checks (issue #26633).+ struct CustomParquetFileReaderFactory {+ store: Arc<dyn ObjectStore>,+ }++ impl Debug for CustomParquetFileReaderFactory {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("DefaultParquetFileReaderFactory")+ .field("store", &self.store)+ .finish()+ }+ }++ impl ParquetFileReaderFactory for CustomParquetFileReaderFactory {+ fn create_reader(+ &self,+ _partition_index: usize,+ _partitioned_file: PartitionedFile,+ _metadata_size_hint: Option<usize>,+ _metrics: &ExecutionPlanMetricsSet,+ ) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>> {+ unimplemented!()+ }+ }++ let custom_factory = Arc::new(CustomParquetFileReaderFactory { store });+ assert!(+ format!("{custom_factory:?}")+ .starts_with("DefaultParquetFileReaderFactory { store:")+ );++ let source = ParquetSource::new(Arc::new(Schema::empty()))+ .with_parquet_file_reader_factory(custom_factory);+ let factory = source.parquet_file_reader_factory().unwrap();+ assert!(!factory.is::<DefaultParquetFileReaderFactory>());+ assert!(!factory.is::<CachedParquetFileReaderFactory>());+ assert!(+ factory+ .downcast_ref::<DefaultParquetFileReaderFactory>()+ .is_none()+ );+ assert!(+ factory+ .downcast_ref::<CachedParquetFileReaderFactory>()+ .is_none()+ );+ assert!(factory.is::<CustomParquetFileReaderFactory>());+ assert!(+ factory+ .downcast_ref::<CustomParquetFileReaderFactory>()+ .is_some()+ );+ }+/// Render a `ParquetSource`'s `fmt_extra` output as a `String` for/// inspection in tests.fn render_fmt_extra(source: &ParquetSource, t: DisplayFormatType) -> String {diff --git a/docs/source/library-user-guide/upgrading/57.0.0.md b/docs/source/library-user-guide/upgrading/57.0.0.mdindex 31a3e3d399..4564a84c28 100644--- a/docs/source/library-user-guide/upgrading/57.0.0.md+++ b/docs/source/library-user-guide/upgrading/57.0.0.md@@ -62,3 +62,18 @@ available without the Spark function library:```sqlSET datafusion.sql_parser.dialect = 'spark';```++### `ParquetFileReaderFactory` adds `Any` as a supertrait++To enable downcasting of `dyn ParquetFileReaderFactory` to concrete factory+types (via `is::<T>()` / `downcast_ref::<T>()`), the+`ParquetFileReaderFactory` trait now has `Any` as a supertrait:++```diff+- pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static++ pub trait ParquetFileReaderFactory: Any + Debug + Send + Sync+```++The trait already required `'static`, which `Any` implies, so existing+implementations need no changes. `ParquetSource::parquet_file_reader_factory`+returns the trait object to downcast.