Skip to content

Apache DataFusion SQL Query Engine

feat: allow downcasting `ParquetFileReaderFactory` (#26635)

Morris9f8982181ec7
diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rs
index 1452bc8ef8..f9a5e11fdd 100644
--- a/datafusion/datasource-parquet/src/reader.rs
+++ b/datafusion/datasource-parquet/src/reader.rs
@@ -55,7 +55,7 @@ pub trait ParquetFileSchemaProvider: Debug + Send + Sync + 'static {
/// such as pre-cached metadata, I/O coalescing, etc.
///
/// See [`DefaultParquetFileReaderFactory`] for a simple implementation.
-pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {
+pub trait ParquetFileReaderFactory: Any + Debug + Send + Sync {
/// Provides an `AsyncFileReader` for reading data from a parquet file specified
///
/// # Notes
@@ -79,6 +79,18 @@ pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {
) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>>;
}
+impl dyn ParquetFileReaderFactory {
+ /// Returns `true` if this factory is of type `T`.
+ pub fn is<T: ParquetFileReaderFactory>(&self) -> bool {
+ (self as &dyn Any).is::<T>()
+ }
+
+ /// Attempts to downcast this factory to a concrete type `T`.
+ pub fn downcast_ref<T: ParquetFileReaderFactory>(&self) -> Option<&T> {
+ (self as &dyn Any).downcast_ref()
+ }
+}
+
/// Default implementation of [`ParquetFileReaderFactory`]
///
/// This implementation:
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index c818cb3512..09388b0963 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -1518,6 +1518,107 @@ mod tests {
assert!(source.filter().is_some());
}
+ #[test]
+ fn parquet_file_reader_factory_downcasts_to_concrete_type() {
+ use crate::CachedParquetFileReaderFactory;
+ use datafusion_datasource::PartitionedFile;
+ use datafusion_execution::cache::cache_manager::FileMetadataCache;
+ use datafusion_execution::cache::default_cache::DefaultCache;
+ use object_store::memory::InMemory;
+ use parquet::arrow::async_reader::AsyncFileReader;
+
+ let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+ let source = ParquetSource::new(Arc::new(Schema::empty()))
+ .with_parquet_file_reader_factory(Arc::new(
+ DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
+ ));
+ let factory = source.parquet_file_reader_factory().unwrap();
+ assert!(factory.is::<DefaultParquetFileReaderFactory>());
+ assert!(
+ factory
+ .downcast_ref::<DefaultParquetFileReaderFactory>()
+ .is_some()
+ );
+ assert!(
+ factory
+ .downcast_ref::<CachedParquetFileReaderFactory>()
+ .is_none()
+ );
+
+ let cache: Arc<FileMetadataCache> = Arc::new(DefaultCache::new(1024));
+ let source = ParquetSource::new(Arc::new(Schema::empty()))
+ .with_parquet_file_reader_factory(Arc::new(
+ CachedParquetFileReaderFactory::new(Arc::clone(&store), cache),
+ ));
+ let factory = source.parquet_file_reader_factory().unwrap();
+ assert!(factory.is::<CachedParquetFileReaderFactory>());
+ assert!(
+ factory
+ .downcast_ref::<CachedParquetFileReaderFactory>()
+ .is_some()
+ );
+ assert!(
+ factory
+ .downcast_ref::<DefaultParquetFileReaderFactory>()
+ .is_none()
+ );
+
+ // A custom factory whose Debug output resembles a built-in factory.
+ // Verifies that concrete-type downcasting avoids false-positive matches
+ // from Debug substring checks (issue #26633).
+ struct CustomParquetFileReaderFactory {
+ store: Arc<dyn ObjectStore>,
+ }
+
+ impl Debug for CustomParquetFileReaderFactory {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("DefaultParquetFileReaderFactory")
+ .field("store", &self.store)
+ .finish()
+ }
+ }
+
+ impl ParquetFileReaderFactory for CustomParquetFileReaderFactory {
+ fn create_reader(
+ &self,
+ _partition_index: usize,
+ _partitioned_file: PartitionedFile,
+ _metadata_size_hint: Option<usize>,
+ _metrics: &ExecutionPlanMetricsSet,
+ ) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>> {
+ unimplemented!()
+ }
+ }
+
+ let custom_factory = Arc::new(CustomParquetFileReaderFactory { store });
+ assert!(
+ format!("{custom_factory:?}")
+ .starts_with("DefaultParquetFileReaderFactory { store:")
+ );
+
+ let source = ParquetSource::new(Arc::new(Schema::empty()))
+ .with_parquet_file_reader_factory(custom_factory);
+ let factory = source.parquet_file_reader_factory().unwrap();
+ assert!(!factory.is::<DefaultParquetFileReaderFactory>());
+ assert!(!factory.is::<CachedParquetFileReaderFactory>());
+ assert!(
+ factory
+ .downcast_ref::<DefaultParquetFileReaderFactory>()
+ .is_none()
+ );
+ assert!(
+ factory
+ .downcast_ref::<CachedParquetFileReaderFactory>()
+ .is_none()
+ );
+ assert!(factory.is::<CustomParquetFileReaderFactory>());
+ assert!(
+ factory
+ .downcast_ref::<CustomParquetFileReaderFactory>()
+ .is_some()
+ );
+ }
+
/// Render a `ParquetSource`'s `fmt_extra` output as a `String` for
/// inspection in tests.
fn render_fmt_extra(source: &ParquetSource, t: DisplayFormatType) -> String {
diff --git a/docs/source/library-user-guide/upgrading/57.0.0.md b/docs/source/library-user-guide/upgrading/57.0.0.md
index 31a3e3d399..4564a84c28 100644
--- a/docs/source/library-user-guide/upgrading/57.0.0.md
+++ b/docs/source/library-user-guide/upgrading/57.0.0.md
@@ -62,3 +62,18 @@ available without the Spark function library:
```sql
SET datafusion.sql_parser.dialect = 'spark';
```
+
+### `ParquetFileReaderFactory` adds `Any` as a supertrait
+
+To enable downcasting of `dyn ParquetFileReaderFactory` to concrete factory
+types (via `is::<T>()` / `downcast_ref::<T>()`), the
+`ParquetFileReaderFactory` trait now has `Any` as a supertrait:
+
+```diff
+- pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static
++ pub trait ParquetFileReaderFactory: Any + Debug + Send + Sync
+```
+
+The trait already required `'static`, which `Any` implies, so existing
+implementations need no changes. `ParquetSource::parquet_file_reader_factory`
+returns the trait object to downcast.