Apache DataFusion SQL Query Engine
feat: type-keyed extensions map for PartitionedFile (#21993)
Adrian Garcia Badaracco7d63fb7e4d85
diff --git a/datafusion-examples/examples/data_io/parquet_advanced_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rsindex 9e69c7f15a..9bdcda265e 100644--- a/datafusion-examples/examples/data_io/parquet_advanced_index.rs+++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs@@ -476,7 +476,7 @@ impl TableProvider for IndexTableProvider {.partitioned_file()// provide the starting access plan to the DataSourceExec by// storing it as "extensions" on PartitionedFile- .with_extensions(Arc::new(access_plan) as _);+ .with_extension(access_plan);// Prepare for scanninglet schema = self.schema();diff --git a/datafusion/common/src/extensions.rs b/datafusion/common/src/extensions.rsnew file mode 100644index 0000000000..d5a21fb656--- /dev/null+++ b/datafusion/common/src/extensions.rs@@ -0,0 +1,211 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! A type-keyed map of opaque, `Arc`'d objects.+//!+//! Used as the backing store for the various `extensions` fields throughout+//! DataFusion (e.g. [`SessionConfig`], [`ExtendedStatistics`],+//! [`PartitionedFile`]) so that independent components can each attach+//! their own data without conflict, each keyed by its concrete Rust type.+//!+//! [`SessionConfig`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/config/struct.SessionConfig.html+//! [`ExtendedStatistics`]: https://docs.rs/datafusion-physical-plan/latest/datafusion_physical_plan/operator_statistics/struct.ExtendedStatistics.html+//! [`PartitionedFile`]: https://docs.rs/datafusion-datasource/latest/datafusion_datasource/struct.PartitionedFile.html++use std::any::{Any, TypeId};+use std::collections::HashMap;+use std::hash::{BuildHasherDefault, Hasher};+use std::sync::Arc;++/// A type-keyed map of opaque `Arc`'d values. Each Rust type `T` occupies+/// its own slot, so independent components can each attach their own data+/// without conflict.+///+/// Cloning is cheap: the backing values are reference-counted.+///+/// # Example+///+/// ```+/// # use std::sync::Arc;+/// # use datafusion_common::extensions::Extensions;+/// struct MyData(u32);+/// struct OtherData(&'static str);+///+/// let mut ext = Extensions::new();+/// ext.insert(MyData(42));+/// ext.insert_arc(Arc::new(OtherData("hello")));+///+/// assert_eq!(ext.get::<MyData>().unwrap().0, 42);+/// assert_eq!(ext.get::<OtherData>().unwrap().0, "hello");+/// ```+#[derive(Debug, Clone, Default)]+pub struct Extensions {+ inner: HashMap<TypeId, Arc<dyn Any + Send + Sync>, BuildHasherDefault<IdHasher>>,+}++impl Extensions {+ /// Create an empty map.+ pub fn new() -> Self {+ Self::default()+ }++ /// Returns true if no extensions are set.+ pub fn is_empty(&self) -> bool {+ self.inner.is_empty()+ }++ /// Number of extensions set.+ pub fn len(&self) -> usize {+ self.inner.len()+ }++ /// Insert an extension keyed by its concrete type `T`. Returns the+ /// previous value of that type, if any.+ ///+ /// The value is wrapped in an [`Arc`] internally. If the caller already+ /// has an `Arc<T>` and wants to avoid an extra allocation, use+ /// [`Self::insert_arc`].+ pub fn insert<T: Any + Send + Sync>(&mut self, value: T) -> Option<Arc<T>> {+ self.insert_arc(Arc::new(value))+ }++ /// Insert an extension keyed by its concrete type `T`, taking an+ /// already-allocated [`Arc<T>`]. Returns the previous value of that type,+ /// if any.+ pub fn insert_arc<T: Any + Send + Sync>(&mut self, value: Arc<T>) -> Option<Arc<T>> {+ self.inner+ .insert(TypeId::of::<T>(), value)+ .map(|p| Arc::downcast::<T>(p).expect("TypeId matches T"))+ }++ /// Insert an already-type-erased value, keyed by its dynamic+ /// [`TypeId`]. Used internally to support APIs that accept+ /// `Arc<dyn Any + Send + Sync>` for backwards compatibility and need+ /// to recover the concrete type for keying.+ ///+ /// New code should use [`Self::insert`] or [`Self::insert_arc`], which+ /// preserve the concrete type at the call site.+ #[deprecated(+ since = "54.0.0",+ note = "use `insert` or `insert_arc`; only retained to support the deprecated `PartitionedFile::with_extensions` shim"+ )]+ pub fn insert_dyn(+ &mut self,+ value: Arc<dyn Any + Send + Sync>,+ ) -> Option<Arc<dyn Any + Send + Sync>> {+ let id = (*value).type_id();+ self.inner.insert(id, value)+ }++ /// Borrow the extension of type `T`, if set.+ pub fn get<T: Any + Send + Sync>(&self) -> Option<&T> {+ self.inner+ .get(&TypeId::of::<T>())+ .and_then(|a| a.downcast_ref::<T>())+ }++ /// Get a cloned `Arc<T>` of the extension, if set.+ pub fn get_arc<T: Any + Send + Sync>(&self) -> Option<Arc<T>> {+ self.inner+ .get(&TypeId::of::<T>())+ .map(|a| Arc::downcast::<T>(Arc::clone(a)).expect("TypeId matches T"))+ }++ /// Returns true if an extension of type `T` is set.+ pub fn contains<T: Any + Send + Sync>(&self) -> bool {+ self.inner.contains_key(&TypeId::of::<T>())+ }++ /// Merge entries from `other` into `self`. Entries in `other` take+ /// precedence over existing entries with the same type.+ pub fn merge(&mut self, other: &Extensions) {+ for (id, ext) in &other.inner {+ self.inner.insert(*id, Arc::clone(ext));+ }+ }+}++/// Hasher specialized for [`TypeId`] keys. Since `TypeId` is already a+/// hash produced by the compiler, we don't need to hash it again — we+/// just store the `u64` it writes and return it unchanged.+#[derive(Default)]+struct IdHasher(u64);++impl Hasher for IdHasher {+ fn write(&mut self, _: &[u8]) {+ unreachable!("TypeId calls write_u64");+ }++ #[inline]+ fn write_u64(&mut self, id: u64) {+ self.0 = id;+ }++ #[inline]+ fn finish(&self) -> u64 {+ self.0+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[derive(Debug, PartialEq)]+ struct A(u32);++ #[derive(Debug, PartialEq)]+ struct B(&'static str);++ #[test]+ fn insert_get_replace() {+ let mut ext = Extensions::new();+ assert!(ext.is_empty());++ ext.insert(A(1));+ ext.insert_arc(Arc::new(B("x")));+ assert_eq!(ext.len(), 2);+ assert_eq!(ext.get::<A>(), Some(&A(1)));+ assert_eq!(ext.get::<B>(), Some(&B("x")));+ assert!(ext.contains::<A>());++ let prev = ext.insert(A(2));+ assert_eq!(prev.as_deref(), Some(&A(1)));+ assert_eq!(ext.get::<A>(), Some(&A(2)));+ }++ #[test]+ #[expect(deprecated)]+ fn insert_dyn_keys_by_concrete_type() {+ let mut ext = Extensions::new();+ let erased: Arc<dyn Any + Send + Sync> = Arc::new(A(7));+ ext.insert_dyn(erased);+ assert_eq!(ext.get::<A>(), Some(&A(7)));+ }++ #[test]+ fn merge_other_wins() {+ let mut a = Extensions::new();+ a.insert(A(1));+ let mut b = Extensions::new();+ b.insert(A(2));+ b.insert(B("hi"));+ a.merge(&b);+ assert_eq!(a.get::<A>(), Some(&A(2)));+ assert_eq!(a.get::<B>(), Some(&B("hi")));+ }+}diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rsindex 996c563f0d..9be0941b5d 100644--- a/datafusion/common/src/lib.rs+++ b/datafusion/common/src/lib.rs@@ -43,6 +43,7 @@ pub mod diagnostic;pub mod display;pub mod encryption;pub mod error;+pub mod extensions;pub mod file_options;pub mod format;pub mod hash_utils;diff --git a/datafusion/core/tests/parquet/custom_reader.rs b/datafusion/core/tests/parquet/custom_reader.rsindex ae11fa9a11..31337ed5d8 100644--- a/datafusion/core/tests/parquet/custom_reader.rs+++ b/datafusion/core/tests/parquet/custom_reader.rs@@ -38,6 +38,7 @@ use bytes::Bytes;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::source::DataSourceExec;use datafusion_datasource_parquet::metadata::DFParquetMetadata;+use datafusion_datasource_parquet::{ParquetAccessPlan, RowGroupAccess};use futures::future::BoxFuture;use futures::{FutureExt, TryFutureExt};use insta::assert_snapshot;@@ -49,6 +50,7 @@ use parquet::arrow::arrow_reader::ArrowReaderOptions;use parquet::arrow::async_reader::AsyncFileReader;use parquet::errors::ParquetError;use parquet::file::metadata::ParquetMetaData;+use parquet::file::properties::WriterProperties;const EXPECTED_USER_DEFINED_METADATA: &str = "some-user-defined-metadata";@@ -71,7 +73,7 @@ async fn route_data_access_ops_to_parquet_file_reader_factory() {.into_iter().map(|meta| {PartitionedFile::new_from_meta(meta)- .with_extensions(Arc::new(String::from(EXPECTED_USER_DEFINED_METADATA)))+ .with_extension(String::from(EXPECTED_USER_DEFINED_METADATA))}).collect();@@ -107,6 +109,82 @@ async fn route_data_access_ops_to_parquet_file_reader_factory() {");}+/// Regression test for the type-keyed extensions map: independent components+/// must be able to attach their own per-file payloads on the same+/// [`PartitionedFile`] without colliding. Here we attach a custom reader+/// payload (the `String` checked by [`InMemoryParquetFileReaderFactory`])+/// *and* a [`ParquetAccessPlan`] that skips the first row group, then verify+/// (a) the factory still sees its payload (its internal `assert_eq!` would+/// fire if the slot got overwritten) and (b) the access plan is honored — so+/// only the second row group's 5 rows come out, not all 10.+#[tokio::test]+async fn custom_payload_and_access_plan_coexist() {+ // Two row groups of 5 rows each: values 0..=4 in row group 0, 5..=9 in+ // row group 1.+ let c1: ArrayRef = Arc::new(Int64Array::from((0..10).collect::<Vec<i64>>()));+ let batch = create_batch(vec![("c1", c1)]);+ let file_schema = batch.schema().clone();++ let in_memory = InMemory::new();+ let mut buf = Vec::<u8>::with_capacity(32 * 1024);+ let props = WriterProperties::builder()+ .set_max_row_group_row_count(Some(5))+ .build();+ let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), Some(props)).unwrap();+ writer.write(&batch).unwrap();+ writer.close().unwrap();++ let location = Path::parse("two-row-groups.parquet").unwrap();+ let size = buf.len() as u64;+ in_memory+ .put(&location, Bytes::from(buf).into())+ .await+ .unwrap();+ let meta = ObjectMeta {+ location,+ last_modified: chrono::DateTime::from(SystemTime::now()),+ size,+ e_tag: None,+ version: None,+ };++ let access_plan =+ ParquetAccessPlan::new(vec![RowGroupAccess::Skip, RowGroupAccess::Scan]);+ let pf = PartitionedFile::new_from_meta(meta)+ .with_extension(String::from(EXPECTED_USER_DEFINED_METADATA))+ .with_extension(access_plan);++ let store: Arc<dyn ObjectStore> = Arc::new(in_memory);+ let source = Arc::new(+ ParquetSource::new(file_schema.clone()).with_parquet_file_reader_factory(+ Arc::new(InMemoryParquetFileReaderFactory(Arc::clone(&store))),+ ),+ );+ let base_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file_group(vec![pf].into())+ .build();+ let parquet_exec = DataSourceExec::from_data_source(base_config);++ let session_ctx = SessionContext::new();+ let read = collect(parquet_exec, session_ctx.task_ctx()).await.unwrap();++ let total: usize = read.iter().map(|b| b.num_rows()).sum();+ assert_eq!(+ total, 5,+ "access plan should have skipped the first row group"+ );++ let values: Vec<i64> = read+ .iter()+ .flat_map(|b| {+ let arr = b.column(0).as_any().downcast_ref::<Int64Array>().unwrap();+ (0..arr.len()).map(|i| arr.value(i)).collect::<Vec<_>>()+ })+ .collect();+ assert_eq!(values, vec![5, 6, 7, 8, 9]);+}+#[derive(Debug)]struct InMemoryParquetFileReaderFactory(Arc<dyn ObjectStore>);@@ -119,12 +197,8 @@ impl ParquetFileReaderFactory for InMemoryParquetFileReaderFactory {metrics: &ExecutionPlanMetricsSet,) -> Result<Box<dyn AsyncFileReader + Send>> {let metadata = partitioned_file- .extensions- .as_ref()+ .extension::<String>().expect("has user defined metadata");- let metadata = metadata- .downcast_ref::<String>()- .expect("has string metadata");assert_eq!(EXPECTED_USER_DEFINED_METADATA, &metadata[..]);diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rsindex 9ff8137687..31be6fd979 100644--- a/datafusion/core/tests/parquet/external_access_plan.rs+++ b/datafusion/core/tests/parquet/external_access_plan.rs@@ -349,7 +349,7 @@ impl TestFull {// add the access plan, if any, as an extensionif let Some(access_plan) = access_plan {- partitioned_file = partitioned_file.with_extensions(Arc::new(access_plan));+ partitioned_file = partitioned_file.with_extension(access_plan);}// Create a DataSourceExec to read the filediff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex bad1c684b4..bbbd298687 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -259,7 +259,7 @@ struct PreparedParquetOpen {partition_index: usize,partitioned_file: PartitionedFile,file_range: Option<datafusion_datasource::FileRange>,- extensions: Option<Arc<dyn std::any::Any + Send + Sync>>,+ extensions: datafusion_datasource::FileExtensions,file_name: String,file_metrics: ParquetFileMetrics,baseline_metrics: BaselineMetrics,@@ -884,7 +884,7 @@ impl FiltersPreparedParquetOpen {// as many row groups as possible based on the metadata and querylet mut row_groups = RowGroupAccessPlanFilter::new(create_initial_plan(&prepared.file_name,- prepared.extensions.clone(),+ &prepared.extensions,rg_metadata.len(),)?);@@ -1540,23 +1540,17 @@ impl ParquetMorselizer {/// Note: file_name is only used for error messagesfn create_initial_plan(file_name: &str,- extensions: Option<Arc<dyn std::any::Any + Send + Sync>>,+ extensions: &datafusion_datasource::FileExtensions,row_group_count: usize,) -> Result<ParquetAccessPlan> {- if let Some(extensions) = extensions {- if let Some(access_plan) = extensions.downcast_ref::<ParquetAccessPlan>() {- let plan_len = access_plan.len();- if plan_len != row_group_count {- return exec_err!(- "Invalid ParquetAccessPlan for {file_name}. Specified {plan_len} row groups, but file has {row_group_count}"- );- }-- // check row group count matches the plan- return Ok(access_plan.clone());- } else {- debug!("DataSourceExec Ignoring unknown extension specified for {file_name}");+ if let Some(access_plan) = extensions.get::<ParquetAccessPlan>() {+ let plan_len = access_plan.len();+ if plan_len != row_group_count {+ return exec_err!(+ "Invalid ParquetAccessPlan for {file_name}. Specified {plan_len} row groups, but file has {row_group_count}"+ );}+ return Ok(access_plan.clone());}// default to scanning all row groups@@ -2513,7 +2507,7 @@ mod test {"test.parquet".to_string(),u64::try_from(data_len).unwrap(),)- .with_extensions(Arc::new(access_plan));+ .with_extension(access_plan);let make_opener = |reverse_scan: bool| {ParquetMorselizerBuilder::new()@@ -2614,7 +2608,7 @@ mod test {"test.parquet".to_string(),u64::try_from(data_len).unwrap(),)- .with_extensions(Arc::new(access_plan));+ .with_extension(access_plan);let make_opener = |reverse_scan: bool| {ParquetMorselizerBuilder::new()diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 787f882908..91e3d22749 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -230,7 +230,7 @@ use parquet::encryption::decrypt::FileDecryptionProperties;/// access_plan.skip(4);/// // provide the plan as extension to the FileScanConfig/// let partitioned_file = PartitionedFile::new("my_file.parquet", 1234)-/// .with_extensions(Arc::new(access_plan));+/// .with_extension(access_plan);/// // create a FileScanConfig to scan this file/// let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), Arc::new(ParquetSource::new(schema())))/// .with_file(partitioned_file).build();diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rsindex a9600271c2..c5f9d72200 100644--- a/datafusion/datasource/src/mod.rs+++ b/datafusion/datasource/src/mod.rs@@ -67,10 +67,18 @@ pub use table_schema::TableSchema;#[expect(deprecated)]pub use statistics::add_row_stats;pub use statistics::compute_all_files_statistics;+use std::any::Any;use std::ops::Range;use std::pin::Pin;use std::sync::Arc;+/// User-defined per-file extension data, keyed by concrete Rust type.+///+/// Re-exported from [`datafusion_common::extensions::Extensions`]; the same+/// type backs `SessionConfig::extensions`, `ExtendedStatistics::extensions`,+/// and other extension fields throughout DataFusion.+pub type FileExtensions = datafusion_common::extensions::Extensions;+/// Stream of files get listed from object store#[deprecated(since = "54.0.0",@@ -148,8 +156,10 @@ pub struct PartitionedFile {/// underlying format (for example, Parquet `sorting_columns`), but it may also be set/// explicitly via [`Self::with_ordering`].pub ordering: Option<LexOrdering>,- /// An optional field for user defined per object metadata- pub extensions: Option<Arc<dyn std::any::Any + Send + Sync>>,+ /// User-defined per-file metadata, keyed by Rust type. Multiple+ /// independent components can each attach their own data here without+ /// conflict — see [`FileExtensions`].+ pub extensions: FileExtensions,/// The estimated size of the parquet metadata, in bytespub metadata_size_hint: Option<usize>,}@@ -169,7 +179,7 @@ impl PartitionedFile {range: None,statistics: None,ordering: None,- extensions: None,+ extensions: FileExtensions::new(),metadata_size_hint: None,}}@@ -182,7 +192,7 @@ impl PartitionedFile {range: None,statistics: None,ordering: None,- extensions: None,+ extensions: FileExtensions::new(),metadata_size_hint: None,}}@@ -201,7 +211,7 @@ impl PartitionedFile {range: Some(FileRange { start, end }),statistics: None,ordering: None,- extensions: None,+ extensions: FileExtensions::new(),metadata_size_hint: None,}.with_range(start, end)@@ -257,14 +267,34 @@ impl PartitionedFile {self}- /// Update the user defined extensions for this file.+ /// Attach a typed user-defined extension to this file. Multiple+ /// independent extensions can be attached, each keyed by its concrete+ /// Rust type. Inserting a value of a type that already has an extension+ /// replaces the previous one.+ ///+ /// This can be used to pass reader-specific information (e.g. a+ /// `ParquetAccessPlan`, or a custom index entry).+ pub fn with_extension<T: Any + Send + Sync>(mut self, value: T) -> Self {+ self.extensions.insert(value);+ self+ }++ /// Borrow the extension of type `T`, if one is attached.+ pub fn extension<T: Any + Send + Sync>(&self) -> Option<&T> {+ self.extensions.get::<T>()+ }++ /// Attach a type-erased extension to this file.///- /// This can be used to pass reader specific information.- pub fn with_extensions(- mut self,- extensions: Arc<dyn std::any::Any + Send + Sync>,- ) -> Self {- self.extensions = Some(extensions);+ /// Kept as a backwards-compatible shim; prefer [`Self::with_extension`]+ /// which keys the extension by its concrete Rust type at the call site.+ #[deprecated(+ since = "54.0.0",+ note = "use `with_extension`; the extension is keyed by its concrete type"+ )]+ pub fn with_extensions(mut self, extensions: Arc<dyn Any + Send + Sync>) -> Self {+ #[expect(deprecated)]+ self.extensions.insert_dyn(extensions);self}@@ -338,7 +368,7 @@ impl From<ObjectMeta> for PartitionedFile {range: None,statistics: None,ordering: None,- extensions: None,+ extensions: FileExtensions::new(),metadata_size_hint: None,}}@@ -535,7 +565,7 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGro}],})),ordering: None,- extensions: None,+ extensions: FileExtensions::new(),metadata_size_hint: None,};files.push(file);diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rsindex 6f60711631..b2917a4583 100644--- a/datafusion/execution/src/config.rs+++ b/datafusion/execution/src/config.rs@@ -15,16 +15,12 @@// specific language governing permissions and limitations// under the License.-use std::{- any::{Any, TypeId},- collections::HashMap,- hash::{BuildHasherDefault, Hasher},- sync::Arc,-};+use std::{collections::HashMap, sync::Arc};use datafusion_common::{Result, ScalarValue,config::{ConfigExtension, ConfigOptions, SpillCompression},+ extensions::Extensions,};/// Configuration options for [`SessionContext`].@@ -99,19 +95,17 @@ pub struct SessionConfig {/// A new copy is created on write, if there are other outstanding/// references to the same options.options: Arc<ConfigOptions>,- /// Opaque extensions.- extensions: AnyMap,+ /// Opaque extensions, keyed by concrete Rust type. See+ /// [`with_extension`](Self::with_extension) and+ /// [`get_extension`](Self::get_extension).+ extensions: Extensions,}impl Default for SessionConfig {fn default() -> Self {Self {options: Arc::new(ConfigOptions::new()),- // Assume no extensions by default.- extensions: HashMap::with_capacity_and_hasher(- 0,- BuildHasherDefault::default(),- ),+ extensions: Extensions::new(),}}}@@ -602,9 +596,7 @@ impl SessionConfig {whereT: Send + Sync + 'static,{- let ext = ext as Arc<dyn Any + Send + Sync + 'static>;- let id = TypeId::of::<T>();- self.extensions.insert(id, ext);+ self.extensions.insert_arc(ext);}/// Get extension, if any for the specified type `T` exists.@@ -614,11 +606,7 @@ impl SessionConfig {whereT: Send + Sync + 'static,{- let id = TypeId::of::<T>();- self.extensions- .get(&id)- .cloned()- .map(|ext| Arc::downcast(ext).expect("TypeId unique"))+ self.extensions.get_arc::<T>()}}@@ -631,34 +619,3 @@ impl From<ConfigOptions> for SessionConfig {}}}--/// Map that holds opaque objects indexed by their type.-///-/// Data is wrapped into an [`Arc`] to enable [`Clone`] while still being [object safe].-///-/// [object safe]: https://doc.rust-lang.org/reference/items/traits.html#object-safety-type AnyMap =- HashMap<TypeId, Arc<dyn Any + Send + Sync + 'static>, BuildHasherDefault<IdHasher>>;--/// Hasher for [`AnyMap`].-///-/// With [`TypeId`]s as keys, there's no need to hash them. They are already hashes themselves, coming from the compiler.-/// The [`IdHasher`] just holds the [`u64`] of the [`TypeId`], and then returns it, instead of doing any bit fiddling.-#[derive(Default)]-struct IdHasher(u64);--impl Hasher for IdHasher {- fn write(&mut self, _: &[u8]) {- unreachable!("TypeId calls write_u64");- }-- #[inline]- fn write_u64(&mut self, id: u64) {- self.0 = id;- }-- #[inline]- fn finish(&self) -> u64 {- self.0- }-}diff --git a/datafusion/physical-plan/src/operator_statistics/mod.rs b/datafusion/physical-plan/src/operator_statistics/mod.rsindex 20266e9768..b577482ba4 100644--- a/datafusion/physical-plan/src/operator_statistics/mod.rs+++ b/datafusion/physical-plan/src/operator_statistics/mod.rs@@ -86,11 +86,10 @@//! let stats = registry.compute(plan.as_ref())?;//! ```-use std::any::{Any, TypeId};-use std::collections::HashMap;use std::fmt::{self, Debug};use std::sync::Arc;+use datafusion_common::extensions::Extensions;use datafusion_common::stats::Precision;use datafusion_common::{Result, Statistics};@@ -128,7 +127,7 @@ pub struct ExtendedStatistics {/// Standard statistics (num_rows, byte_size, column stats)base: Arc<Statistics>,/// Type-erased extensions for custom statistics- extensions: HashMap<TypeId, Arc<dyn Any + Send + Sync>>,+ extensions: Extensions,}impl ExtendedStatistics {@@ -136,7 +135,7 @@ impl ExtendedStatistics {pub fn new(base: Statistics) -> Self {Self {base: Arc::new(base),- extensions: HashMap::new(),+ extensions: Extensions::new(),}}@@ -144,7 +143,7 @@ impl ExtendedStatistics {pub fn new_arc(base: Arc<Statistics>) -> Self {Self {base,- extensions: HashMap::new(),+ extensions: Extensions::new(),}}@@ -160,26 +159,22 @@ impl ExtendedStatistics {/// Get a reference to a custom statistics extension by type.pub fn get_extension<T: 'static + Send + Sync>(&self) -> Option<&T> {- self.extensions- .get(&TypeId::of::<T>())- .and_then(|ext| ext.downcast_ref())+ self.extensions.get::<T>()}/// Set a custom statistics extension.pub fn set_extension<T: 'static + Send + Sync>(&mut self, value: T) {- self.extensions.insert(TypeId::of::<T>(), Arc::new(value));+ self.extensions.insert(value);}/// Check if an extension of the given type exists.pub fn has_extension<T: 'static + Send + Sync>(&self) -> bool {- self.extensions.contains_key(&TypeId::of::<T>())+ self.extensions.contains::<T>()}/// Merge extensions from another ExtendedStatistics (other's extensions take precedence).pub fn merge_extensions(&mut self, other: &ExtendedStatistics) {- for (type_id, ext) in &other.extensions {- self.extensions.insert(*type_id, Arc::clone(ext));- }+ self.extensions.merge(&other.extensions);}}diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.mdindex e6885fbdfa..7911bbdc50 100644--- a/docs/source/library-user-guide/upgrading/54.0.0.md+++ b/docs/source/library-user-guide/upgrading/54.0.0.md@@ -515,6 +515,48 @@ SELECT CAST(approx_percentile_cont(quantity, 0.5) AS BIGINT) FROM orders;[#21074]: https://github.com/apache/datafusion/pull/21074+### `PartitionedFile::extensions` is now a type-keyed map++`PartitionedFile.extensions` previously held a single+`Option<Arc<dyn Any + Send + Sync>>` slot, so two independent components+could not both attach data to the same file without colliding. The field+is now a `FileExtensions` (a re-export of+`datafusion_common::extensions::Extensions`), a map keyed by concrete Rust+type. Each type occupies its own slot, so multiple consumers (e.g. a+`ParquetAccessPlan` and a custom index entry) can coexist on a single+`PartitionedFile`.++The previous `with_extensions(Arc<dyn Any + Send + Sync>)` builder is+deprecated (it still works, keyed by the value's dynamic `TypeId`) in+favor of a typed variant:++```diff+-let pf = PartitionedFile::new(path, size)+- .with_extensions(Arc::new(access_plan));++let pf = PartitionedFile::new(path, size)++ .with_extension(access_plan);+```++Reading an extension no longer requires a manual downcast:++```diff+-let access_plan = partitioned_file+- .extensions+- .as_ref()+- .and_then(|ext| ext.downcast_ref::<ParquetAccessPlan>());++let access_plan = partitioned_file.extension::<ParquetAccessPlan>();+```++The `FileExtensions` API is `insert` / `insert_arc` / `get` / `get_arc`+/ `contains` / `merge`, all generic over the concrete type `T`. Values+are stored as `Arc<T>` so the map remains cheap to clone.++**Who is affected:**++- Code that constructs `PartitionedFile` and calls `.with_extensions(...)`.+- Custom `ParquetFileReaderFactory` implementations or other consumers that+ read `partitioned_file.extensions` and downcast manually.+### `Box<C>` and `Arc<C>` `TreeNodeContainer` impls now require `C: Default`The generic `TreeNodeContainer` implementations for `Box<C>` and `Arc<C>` now