Apache DataFusion SQL Query Engine
Refactor cache APIs to support ordering information (#19597)
Adrian Garcia Badaracco0cf45cae9c63
diff --git a/Cargo.lock b/Cargo.lockindex 2ee907b30c..5dc1f5e1f9 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2120,6 +2120,7 @@ dependencies = ["dashmap","datafusion-common","datafusion-expr",+ "datafusion-physical-expr-common","futures","insta","log",diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rsindex 8a6ad448d8..6a97c5355f 100644--- a/datafusion-cli/src/functions.rs+++ b/datafusion-cli/src/functions.rs@@ -831,14 +831,14 @@ impl TableFunctionImpl for ListFilesCacheFunc {.map(|t| t.duration_since(now).as_millis() as i64),);- for meta in entry.metas.iter() {+ for meta in entry.metas.files.iter() {file_path_arr.push(meta.location.to_string());file_modified_arr.push(meta.last_modified.timestamp_millis());file_size_bytes_arr.push(meta.size);etag_arr.push(meta.e_tag.clone());version_arr.push(meta.version.clone());}- current_offset += entry.metas.len() as i32;+ current_offset += entry.metas.files.len() as i32;offsets.push(current_offset);}}diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex a175d47f4d..31fb029fd5 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -710,31 +710,42 @@ impl ListingTable {store: &Arc<dyn ObjectStore>,part_file: &PartitionedFile,) -> datafusion_common::Result<Arc<Statistics>> {- match self+ use datafusion_execution::cache::cache_manager::CachedFileMetadata;++ // Check cache first+ if let Some(cached) = self.collected_statistics- .get_with_extra(&part_file.object_meta.location, &part_file.object_meta)+ .get(&part_file.object_meta.location){- Some(statistics) => Ok(statistics),- None => {- let statistics = self- .options- .format- .infer_stats(- ctx,- store,- Arc::clone(&self.file_schema),- &part_file.object_meta,- )- .await?;- let statistics = Arc::new(statistics);- self.collected_statistics.put_with_extra(- &part_file.object_meta.location,- Arc::clone(&statistics),- &part_file.object_meta,- );- Ok(statistics)+ // Validate that cached entry is still valid+ if cached.is_valid_for(&part_file.object_meta) {+ return Ok(cached.statistics);}}++ // Cache miss or invalid - infer statistics+ let statistics = self+ .options+ .format+ .infer_stats(+ ctx,+ store,+ Arc::clone(&self.file_schema),+ &part_file.object_meta,+ )+ .await?;+ let statistics = Arc::new(statistics);++ // Store in cache+ self.collected_statistics.put(+ &part_file.object_meta.location,+ CachedFileMetadata::new(+ part_file.object_meta.clone(),+ Arc::clone(&statistics),+ None, // No ordering information in this PR+ ),+ );+ Ok(statistics)}}diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rsindex 8b11ba64ae..a4ba196ae4 100644--- a/datafusion/datasource-parquet/src/metadata.rs+++ b/datafusion/datasource-parquet/src/metadata.rs@@ -31,7 +31,9 @@ use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, DataFusionError, Result, ScalarValue, Statistics,};-use datafusion_execution::cache::cache_manager::{FileMetadata, FileMetadataCache};+use datafusion_execution::cache::cache_manager::{+ CachedFileMetadataEntry, FileMetadata, FileMetadataCache,+};use datafusion_functions_aggregate_common::min_max::{MaxAccumulator, MinAccumulator};use datafusion_physical_plan::Accumulator;use log::debug;@@ -125,19 +127,15 @@ impl<'a> DFParquetMetadata<'a> {!cfg!(feature = "parquet_encryption") || decryption_properties.is_none();if cache_metadata- && let Some(parquet_metadata) = file_metadata_cache- .as_ref()- .and_then(|file_metadata_cache| file_metadata_cache.get(object_meta))- .and_then(|file_metadata| {- file_metadata- .as_any()- .downcast_ref::<CachedParquetMetaData>()- .map(|cached_parquet_metadata| {- Arc::clone(cached_parquet_metadata.parquet_metadata())- })- })+ && let Some(file_metadata_cache) = file_metadata_cache.as_ref()+ && let Some(cached) = file_metadata_cache.get(&object_meta.location)+ && cached.is_valid_for(object_meta)+ && let Some(cached_parquet) = cached+ .file_metadata+ .as_any()+ .downcast_ref::<CachedParquetMetaData>(){- return Ok(parquet_metadata);+ return Ok(Arc::clone(cached_parquet.parquet_metadata()));}let mut reader =@@ -163,8 +161,11 @@ impl<'a> DFParquetMetadata<'a> {if cache_metadata && let Some(file_metadata_cache) = file_metadata_cache {file_metadata_cache.put(- object_meta,- Arc::new(CachedParquetMetaData::new(Arc::clone(&metadata))),+ &object_meta.location,+ CachedFileMetadataEntry::new(+ (*object_meta).clone(),+ Arc::new(CachedParquetMetaData::new(Arc::clone(&metadata))),+ ),);}diff --git a/datafusion/datasource/src/url.rs b/datafusion/datasource/src/url.rsindex 2428275ac3..678bd280fc 100644--- a/datafusion/datasource/src/url.rs+++ b/datafusion/datasource/src/url.rs@@ -19,6 +19,7 @@ use std::sync::Arc;use datafusion_common::{DataFusionError, Result, TableReference};use datafusion_execution::cache::TableScopedPath;+use datafusion_execution::cache::cache_manager::CachedFileList;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_session::Session;@@ -396,42 +397,35 @@ async fn list_with_cache<'b>(.map(|res| res.map_err(|e| DataFusionError::ObjectStore(Box::new(e)))).boxed()),Some(cache) => {- // Convert prefix to Option<Path> for cache lookup- let prefix_filter = prefix.cloned();+ // Build the filter prefix (only Some if prefix was requested)+ let filter_prefix = prefix.is_some().then(|| full_prefix.clone());let table_scoped_base_path = TableScopedPath {table: table_ref.cloned(),path: table_base_path.clone(),};- // Try cache lookup with optional prefix filter- let vec = if let Some(res) =- cache.get_with_extra(&table_scoped_base_path, &prefix_filter)- {+ // Try cache lookup - get returns CachedFileList+ let vec = if let Some(cached) = cache.get(&table_scoped_base_path) {debug!("Hit list files cache");- res.as_ref().clone()+ cached.files_matching_prefix(&filter_prefix)} else {// Cache miss - always list and cache the full table// This ensures we have complete data for future prefix queries- let vec = store+ let mut vec = store.list(Some(table_base_path)).try_collect::<Vec<ObjectMeta>>().await?;- cache.put(&table_scoped_base_path, Arc::new(vec.clone()));-- // If a prefix filter was requested, apply it to the results- if prefix.is_some() {- let full_prefix_str = full_prefix.as_ref();- vec.into_iter()- .filter(|meta| {- meta.location.as_ref().starts_with(full_prefix_str)- })- .collect()- } else {- vec- }+ vec.shrink_to_fit(); // Right-size before caching+ let cached: CachedFileList = vec.into();+ let result = cached.files_matching_prefix(&filter_prefix);+ cache.put(&table_scoped_base_path, cached);+ result};- Ok(futures::stream::iter(vec.into_iter().map(Ok)).boxed())+ Ok(+ futures::stream::iter(Arc::unwrap_or_clone(vec).into_iter().map(Ok))+ .boxed(),+ )}}}@@ -531,6 +525,7 @@ mod tests {use std::any::Any;use std::collections::HashMap;use std::ops::Range;+ use std::sync::Arc;use tempfile::tempdir;#[test]diff --git a/datafusion/execution/Cargo.toml b/datafusion/execution/Cargo.tomlindex ca1fba07ca..4d390a8ccc 100644--- a/datafusion/execution/Cargo.toml+++ b/datafusion/execution/Cargo.toml@@ -55,6 +55,7 @@ chrono = { workspace = true }dashmap = { workspace = true }datafusion-common = { workspace = true, default-features = false }datafusion-expr = { workspace = true, default-features = false }+datafusion-physical-expr-common = { workspace = true, default-features = false }futures = { workspace = true }log = { workspace = true }object_store = { workspace = true, features = ["fs"] }diff --git a/datafusion/execution/src/cache/cache_manager.rs b/datafusion/execution/src/cache/cache_manager.rsindex 162074d909..4cc5586440 100644--- a/datafusion/execution/src/cache/cache_manager.rs+++ b/datafusion/execution/src/cache/cache_manager.rs@@ -15,18 +15,21 @@// specific language governing permissions and limitations// under the License.+use crate::cache::CacheAccessor;+use crate::cache::DefaultListFilesCache;use crate::cache::cache_unit::DefaultFilesMetadataCache;use crate::cache::list_files_cache::ListFilesEntry;use crate::cache::list_files_cache::TableScopedPath;-use crate::cache::{CacheAccessor, DefaultListFilesCache};use datafusion_common::TableReference;use datafusion_common::stats::Precision;use datafusion_common::{Result, Statistics};+use datafusion_physical_expr_common::sort_expr::LexOrdering;use object_store::ObjectMeta;use object_store::path::Path;use std::any::Any;use std::collections::HashMap;use std::fmt::{Debug, Formatter};+use std::ops::Deref;use std::sync::Arc;use std::time::Duration;@@ -34,16 +37,61 @@ pub use super::list_files_cache::{DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, DEFAULT_LIST_FILES_CACHE_TTL,};-/// A cache for [`Statistics`].+/// Cached metadata for a file, including statistics and ordering.+///+/// This struct embeds the [`ObjectMeta`] used for cache validation,+/// along with the cached statistics and ordering information.+#[derive(Debug, Clone)]+pub struct CachedFileMetadata {+ /// File metadata used for cache validation (size, last_modified).+ pub meta: ObjectMeta,+ /// Cached statistics for the file, if available.+ pub statistics: Arc<Statistics>,+ /// Cached ordering for the file.+ pub ordering: Option<LexOrdering>,+}++impl CachedFileMetadata {+ /// Create a new cached file metadata entry.+ pub fn new(+ meta: ObjectMeta,+ statistics: Arc<Statistics>,+ ordering: Option<LexOrdering>,+ ) -> Self {+ Self {+ meta,+ statistics,+ ordering,+ }+ }++ /// Check if this cached entry is still valid for the given metadata.+ ///+ /// Returns true if the file size and last modified time match.+ pub fn is_valid_for(&self, current_meta: &ObjectMeta) -> bool {+ self.meta.size == current_meta.size+ && self.meta.last_modified == current_meta.last_modified+ }+}++/// A cache for file statistics and orderings.+///+/// This cache stores [`CachedFileMetadata`] which includes:+/// - File metadata for validation (size, last_modified)+/// - Statistics for the file+/// - Ordering information for the file////// If enabled via [`CacheManagerConfig::with_files_statistics_cache`] this/// cache avoids inferring the same file statistics repeatedly during the/// session lifetime.///+/// The typical usage pattern is:+/// 1. Call `get(path)` to check for cached value+/// 2. If `Some(cached)`, validate with `cached.is_valid_for(¤t_meta)`+/// 3. If invalid or missing, compute new value and call `put(path, new_value)`+////// See [`crate::runtime_env::RuntimeEnv`] for more details-pub trait FileStatisticsCache:- CacheAccessor<Path, Arc<Statistics>, Extra = ObjectMeta>-{+pub trait FileStatisticsCache: CacheAccessor<Path, CachedFileMetadata> {/// Retrieves the information about the entries currently cached.fn list_entries(&self) -> HashMap<Path, FileStatisticsCacheEntry>;}@@ -61,6 +109,63 @@ pub struct FileStatisticsCacheEntry {pub table_size_bytes: Precision<usize>,/// Size of the statistics entry, in bytes.pub statistics_size_bytes: usize,+ /// Whether ordering information is cached for this file.+ pub has_ordering: bool,+}++/// Cached file listing.+///+/// TTL expiration is handled internally by the cache implementation.+#[derive(Debug, Clone, PartialEq)]+pub struct CachedFileList {+ /// The cached file list.+ pub files: Arc<Vec<ObjectMeta>>,+}++impl CachedFileList {+ /// Create a new cached file list.+ pub fn new(files: Vec<ObjectMeta>) -> Self {+ Self {+ files: Arc::new(files),+ }+ }++ /// Filter the files by prefix.+ fn filter_by_prefix(&self, prefix: &Option<Path>) -> Vec<ObjectMeta> {+ match prefix {+ Some(prefix) => self+ .files+ .iter()+ .filter(|meta| meta.location.as_ref().starts_with(prefix.as_ref()))+ .cloned()+ .collect(),+ None => self.files.as_ref().clone(),+ }+ }++ /// Returns files matching the given prefix.+ ///+ /// When prefix is `None`, returns a clone of the `Arc` (no data copy).+ /// When filtering is needed, returns a new `Arc` with filtered results (clones each matching [`ObjectMeta`]).+ pub fn files_matching_prefix(&self, prefix: &Option<Path>) -> Arc<Vec<ObjectMeta>> {+ match prefix {+ None => Arc::clone(&self.files),+ Some(p) => Arc::new(self.filter_by_prefix(&Some(p.clone()))),+ }+ }+}++impl Deref for CachedFileList {+ type Target = Arc<Vec<ObjectMeta>>;+ fn deref(&self) -> &Self::Target {+ &self.files+ }+}++impl From<Vec<ObjectMeta>> for CachedFileList {+ fn from(files: Vec<ObjectMeta>) -> Self {+ Self::new(files)+ }}/// Cache for storing the [`ObjectMeta`]s that result from listing a path@@ -70,21 +175,12 @@ pub struct FileStatisticsCacheEntry {/// especially when done over remote object stores.////// The cache key is always the table's base path, ensuring a stable cache key.-/// The `Extra` type is `Option<Path>`, representing an optional prefix filter-/// (relative to the table base path) for partition-aware lookups.-///-/// When `get_with_extra(key, Some(prefix))` is called:-/// - The cache entry for `key` (table base path) is fetched-/// - Results are filtered to only include files matching `key/prefix`-/// - Filtered results are returned without making a storage call+/// The cached value is a [`CachedFileList`] containing the files and a timestamp.///-/// This enables efficient partition pruning: a single cached listing of the-/// full table can serve queries for any partition subset.+/// Partition filtering is done after retrieval using [`CachedFileList::files_matching_prefix`].////// See [`crate::runtime_env::RuntimeEnv`] for more details.-pub trait ListFilesCache:- CacheAccessor<TableScopedPath, Arc<Vec<ObjectMeta>>, Extra = Option<Path>>-{+pub trait ListFilesCache: CacheAccessor<TableScopedPath, CachedFileList> {/// Returns the cache's memory limit in bytes.fn cache_limit(&self) -> usize;@@ -121,9 +217,44 @@ pub trait FileMetadata: Any + Send + Sync {fn extra_info(&self) -> HashMap<String, String>;}+/// Cached file metadata entry with validation information.+#[derive(Clone)]+pub struct CachedFileMetadataEntry {+ /// File metadata used for cache validation (size, last_modified).+ pub meta: ObjectMeta,+ /// The cached file metadata.+ pub file_metadata: Arc<dyn FileMetadata>,+}++impl CachedFileMetadataEntry {+ /// Create a new cached file metadata entry.+ pub fn new(meta: ObjectMeta, file_metadata: Arc<dyn FileMetadata>) -> Self {+ Self {+ meta,+ file_metadata,+ }+ }++ /// Check if this cached entry is still valid for the given metadata.+ pub fn is_valid_for(&self, current_meta: &ObjectMeta) -> bool {+ self.meta.size == current_meta.size+ && self.meta.last_modified == current_meta.last_modified+ }+}++impl Debug for CachedFileMetadataEntry {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("CachedFileMetadataEntry")+ .field("meta", &self.meta)+ .field("memory_size", &self.file_metadata.memory_size())+ .finish()+ }+}+/// Cache for file-embedded metadata.///-/// This cache stores per-file metadata in the form of [`FileMetadata`],+/// This cache stores per-file metadata in the form of [`CachedFileMetadataEntry`],+/// which includes the [`ObjectMeta`] for validation.////// For example, the built in [`ListingTable`] uses this cache to avoid parsing/// Parquet footers multiple times for the same file.@@ -132,12 +263,15 @@ pub trait FileMetadata: Any + Send + Sync {/// and users can also provide their own implementations to implement custom/// caching strategies.///+/// The typical usage pattern is:+/// 1. Call `get(path)` to check for cached value+/// 2. If `Some(cached)`, validate with `cached.is_valid_for(¤t_meta)`+/// 3. If invalid or missing, compute new value and call `put(path, new_value)`+////// See [`crate::runtime_env::RuntimeEnv`] for more details.////// [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html-pub trait FileMetadataCache:- CacheAccessor<ObjectMeta, Arc<dyn FileMetadata>, Extra = ObjectMeta>-{+pub trait FileMetadataCache: CacheAccessor<Path, CachedFileMetadataEntry> {/// Returns the cache's memory limit in bytes.fn cache_limit(&self) -> usize;diff --git a/datafusion/execution/src/cache/cache_unit.rs b/datafusion/execution/src/cache/cache_unit.rsindex 5351df449a..9362d07841 100644--- a/datafusion/execution/src/cache/cache_unit.rs+++ b/datafusion/execution/src/cache/cache_unit.rs@@ -16,189 +16,339 @@// under the License.use std::collections::HashMap;-use std::sync::Arc;use crate::cache::CacheAccessor;-use crate::cache::cache_manager::{FileStatisticsCache, FileStatisticsCacheEntry};--use datafusion_common::Statistics;+use crate::cache::cache_manager::{+ CachedFileMetadata, FileStatisticsCache, FileStatisticsCacheEntry,+};use dashmap::DashMap;-use object_store::ObjectMeta;use object_store::path::Path;pub use crate::cache::DefaultFilesMetadataCache;/// Default implementation of [`FileStatisticsCache`]///-/// Stores collected statistics for files+/// Stores cached file metadata (statistics and orderings) for files.+///+/// The typical usage pattern is:+/// 1. Call `get(path)` to check for cached value+/// 2. If `Some(cached)`, validate with `cached.is_valid_for(¤t_meta)`+/// 3. If invalid or missing, compute new value and call `put(path, new_value)`///-/// Cache is invalided when file size or last modification has changed+/// Uses DashMap for lock-free concurrent access.////// [`FileStatisticsCache`]: crate::cache::cache_manager::FileStatisticsCache#[derive(Default)]pub struct DefaultFileStatisticsCache {- statistics: DashMap<Path, (ObjectMeta, Arc<Statistics>)>,+ cache: DashMap<Path, CachedFileMetadata>,}-impl FileStatisticsCache for DefaultFileStatisticsCache {- fn list_entries(&self) -> HashMap<Path, FileStatisticsCacheEntry> {- let mut entries = HashMap::<Path, FileStatisticsCacheEntry>::new();-- for entry in &self.statistics {- let path = entry.key();- let (object_meta, stats) = entry.value();- entries.insert(- path.clone(),- FileStatisticsCacheEntry {- object_meta: object_meta.clone(),- num_rows: stats.num_rows,- num_columns: stats.column_statistics.len(),- table_size_bytes: stats.total_byte_size,- statistics_size_bytes: 0, // TODO: set to the real size in the future- },- );- }-- entries- }-}--impl CacheAccessor<Path, Arc<Statistics>> for DefaultFileStatisticsCache {- type Extra = ObjectMeta;-- /// Get `Statistics` for file location.- fn get(&self, k: &Path) -> Option<Arc<Statistics>> {- self.statistics- .get(k)- .map(|s| Some(Arc::clone(&s.value().1)))- .unwrap_or(None)- }-- /// Get `Statistics` for file location. Returns None if file has changed or not found.- fn get_with_extra(&self, k: &Path, e: &Self::Extra) -> Option<Arc<Statistics>> {- self.statistics- .get(k)- .map(|s| {- let (saved_meta, statistics) = s.value();- if saved_meta.size != e.size- || saved_meta.last_modified != e.last_modified- {- // file has changed- None- } else {- Some(Arc::clone(statistics))- }- })- .unwrap_or(None)+impl CacheAccessor<Path, CachedFileMetadata> for DefaultFileStatisticsCache {+ fn get(&self, key: &Path) -> Option<CachedFileMetadata> {+ self.cache.get(key).map(|entry| entry.value().clone())}- /// Save collected file statistics- fn put(&self, _key: &Path, _value: Arc<Statistics>) -> Option<Arc<Statistics>> {- panic!("Put cache in DefaultFileStatisticsCache without Extra not supported.")+ fn put(&self, key: &Path, value: CachedFileMetadata) -> Option<CachedFileMetadata> {+ self.cache.insert(key.clone(), value)}- fn put_with_extra(- &self,- key: &Path,- value: Arc<Statistics>,- e: &Self::Extra,- ) -> Option<Arc<Statistics>> {- self.statistics- .insert(key.clone(), (e.clone(), value))- .map(|x| x.1)- }-- fn remove(&self, k: &Path) -> Option<Arc<Statistics>> {- self.statistics.remove(k).map(|x| x.1.1)+ fn remove(&self, k: &Path) -> Option<CachedFileMetadata> {+ self.cache.remove(k).map(|(_, entry)| entry)}fn contains_key(&self, k: &Path) -> bool {- self.statistics.contains_key(k)+ self.cache.contains_key(k)}fn len(&self) -> usize {- self.statistics.len()+ self.cache.len()}fn clear(&self) {- self.statistics.clear()+ self.cache.clear();}+fn name(&self) -> String {"DefaultFileStatisticsCache".to_string()}}+impl FileStatisticsCache for DefaultFileStatisticsCache {+ fn list_entries(&self) -> HashMap<Path, FileStatisticsCacheEntry> {+ let mut entries = HashMap::<Path, FileStatisticsCacheEntry>::new();++ for entry in self.cache.iter() {+ let path = entry.key();+ let cached = entry.value();+ entries.insert(+ path.clone(),+ FileStatisticsCacheEntry {+ object_meta: cached.meta.clone(),+ num_rows: cached.statistics.num_rows,+ num_columns: cached.statistics.column_statistics.len(),+ table_size_bytes: cached.statistics.total_byte_size,+ statistics_size_bytes: 0, // TODO: set to the real size in the future+ has_ordering: cached.ordering.is_some(),+ },+ );+ }++ entries+ }+}+#[cfg(test)]mod tests {use super::*;use crate::cache::CacheAccessor;- use crate::cache::cache_manager::{FileStatisticsCache, FileStatisticsCacheEntry};- use crate::cache::cache_unit::DefaultFileStatisticsCache;+ use crate::cache::cache_manager::{+ CachedFileMetadata, FileStatisticsCache, FileStatisticsCacheEntry,+ };+ use arrow::array::RecordBatch;use arrow::datatypes::{DataType, Field, Schema, TimeUnit};use chrono::DateTime;use datafusion_common::Statistics;use datafusion_common::stats::Precision;+ use datafusion_expr::ColumnarValue;+ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+ use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use object_store::ObjectMeta;use object_store::path::Path;+ use std::sync::Arc;- #[test]- fn test_statistics_cache() {- let meta = ObjectMeta {- location: Path::from("test"),+ fn create_test_meta(path: &str, size: u64) -> ObjectMeta {+ ObjectMeta {+ location: Path::from(path),last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00").unwrap().into(),- size: 1024,+ size,e_tag: None,version: None,- };+ }+ }++ #[test]+ fn test_statistics_cache() {+ let meta = create_test_meta("test", 1024);+ let cache = DefaultFileStatisticsCache::default();++ let schema = Schema::new(vec![Field::new(+ "test_column",+ DataType::Timestamp(TimeUnit::Second, None),+ false,+ )]);++ // Cache miss+ assert!(cache.get(&meta.location).is_none());++ // Put a value+ let cached_value = CachedFileMetadata::new(+ meta.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&meta.location, cached_value);++ // Cache hit+ let result = cache.get(&meta.location);+ assert!(result.is_some());+ let cached = result.unwrap();+ assert!(cached.is_valid_for(&meta));++ // File size changed - validation should fail+ let meta2 = create_test_meta("test", 2048);+ let cached = cache.get(&meta2.location).unwrap();+ assert!(!cached.is_valid_for(&meta2));++ // Update with new value+ let cached_value2 = CachedFileMetadata::new(+ meta2.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&meta2.location, cached_value2);++ // Test list_entries+ let entries = cache.list_entries();+ assert_eq!(entries.len(), 1);+ let entry = entries.get(&Path::from("test")).unwrap();+ assert_eq!(entry.object_meta.size, 2048); // Should be updated value+ }++ #[derive(Clone, Debug, PartialEq, Eq, Hash)]+ struct MockExpr {}++ impl std::fmt::Display for MockExpr {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "MockExpr")+ }+ }++ impl PhysicalExpr for MockExpr {+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }++ fn data_type(+ &self,+ _input_schema: &Schema,+ ) -> datafusion_common::Result<DataType> {+ Ok(DataType::Int32)+ }++ fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result<bool> {+ Ok(false)+ }++ fn evaluate(+ &self,+ _batch: &RecordBatch,+ ) -> datafusion_common::Result<ColumnarValue> {+ unimplemented!()+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> datafusion_common::Result<Arc<dyn PhysicalExpr>> {+ assert!(children.is_empty());+ Ok(self)+ }++ fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "MockExpr")+ }+ }++ fn ordering() -> LexOrdering {+ let expr = Arc::new(MockExpr {}) as Arc<dyn PhysicalExpr>;+ LexOrdering::new(vec![PhysicalSortExpr::new_default(expr)]).unwrap()+ }++ #[test]+ fn test_ordering_cache() {+ let meta = create_test_meta("test.parquet", 100);let cache = DefaultFileStatisticsCache::default();- assert!(cache.get_with_extra(&meta.location, &meta).is_none());-- cache.put_with_extra(- &meta.location,- Statistics::new_unknown(&Schema::new(vec![Field::new(- "test_column",- DataType::Timestamp(TimeUnit::Second, None),- false,- )]))- .into(),- &meta,++ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);++ // Cache statistics with no ordering+ let cached_value = CachedFileMetadata::new(+ meta.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None, // No ordering yet);- assert!(cache.get_with_extra(&meta.location, &meta).is_some());-- // file size changed- let mut meta2 = meta.clone();- meta2.size = 2048;- assert!(cache.get_with_extra(&meta2.location, &meta2).is_none());-- // file last_modified changed- let mut meta2 = meta.clone();- meta2.last_modified = DateTime::parse_from_rfc3339("2022-09-27T22:40:00+02:00")- .unwrap()- .into();- assert!(cache.get_with_extra(&meta2.location, &meta2).is_none());-- // different file- let mut meta2 = meta.clone();- meta2.location = Path::from("test2");- assert!(cache.get_with_extra(&meta2.location, &meta2).is_none());-- // test the list_entries method+ cache.put(&meta.location, cached_value);++ let result = cache.get(&meta.location).unwrap();+ assert!(result.ordering.is_none());++ // Update to add ordering+ let mut cached = cache.get(&meta.location).unwrap();+ if cached.is_valid_for(&meta) && cached.ordering.is_none() {+ cached.ordering = Some(ordering());+ }+ cache.put(&meta.location, cached);++ let result2 = cache.get(&meta.location).unwrap();+ assert!(result2.ordering.is_some());++ // Verify list_entries shows has_ordering = true+ let entries = cache.list_entries();+ assert_eq!(entries.len(), 1);+ assert!(entries.get(&meta.location).unwrap().has_ordering);+ }++ #[test]+ fn test_cache_invalidation_on_file_modification() {+ let cache = DefaultFileStatisticsCache::default();+ let path = Path::from("test.parquet");+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);++ let meta_v1 = create_test_meta("test.parquet", 100);++ // Cache initial value+ let cached_value = CachedFileMetadata::new(+ meta_v1.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&path, cached_value);++ // File modified (size changed)+ let meta_v2 = create_test_meta("test.parquet", 200);++ let cached = cache.get(&path).unwrap();+ // Should not be valid for new meta+ assert!(!cached.is_valid_for(&meta_v2));++ // Compute new value and update+ let new_cached = CachedFileMetadata::new(+ meta_v2.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&path, new_cached);++ // Should have new metadata+ let result = cache.get(&path).unwrap();+ assert_eq!(result.meta.size, 200);+ }++ #[test]+ fn test_list_entries() {+ let cache = DefaultFileStatisticsCache::default();+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);++ let meta1 = create_test_meta("test1.parquet", 100);++ let cached_value = CachedFileMetadata::new(+ meta1.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&meta1.location, cached_value);+ let meta2 = create_test_meta("test2.parquet", 200);+ let cached_value = CachedFileMetadata::new(+ meta2.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ Some(ordering()),+ );+ cache.put(&meta2.location, cached_value);+let entries = cache.list_entries();assert_eq!(entries,- HashMap::from([(- Path::from("test"),- FileStatisticsCacheEntry {- object_meta: meta.clone(),- num_rows: Precision::Absent,- num_columns: 1,- table_size_bytes: Precision::Absent,- statistics_size_bytes: 0,- }- )])+ HashMap::from([+ (+ Path::from("test1.parquet"),+ FileStatisticsCacheEntry {+ object_meta: meta1,+ num_rows: Precision::Absent,+ num_columns: 1,+ table_size_bytes: Precision::Absent,+ statistics_size_bytes: 0,+ has_ordering: false,+ }+ ),+ (+ Path::from("test2.parquet"),+ FileStatisticsCacheEntry {+ object_meta: meta2,+ num_rows: Precision::Absent,+ num_columns: 1,+ table_size_bytes: Precision::Absent,+ statistics_size_bytes: 0,+ has_ordering: true,+ }+ ),+ ]));}}diff --git a/datafusion/execution/src/cache/file_metadata_cache.rs b/datafusion/execution/src/cache/file_metadata_cache.rsindex c7a24dd878..5e899d7dd9 100644--- a/datafusion/execution/src/cache/file_metadata_cache.rs+++ b/datafusion/execution/src/cache/file_metadata_cache.rs@@ -15,22 +15,19 @@// specific language governing permissions and limitations// under the License.-use std::{- collections::HashMap,- sync::{Arc, Mutex},-};+use std::{collections::HashMap, sync::Mutex};-use object_store::{ObjectMeta, path::Path};+use object_store::path::Path;use crate::cache::{CacheAccessor,- cache_manager::{FileMetadata, FileMetadataCache, FileMetadataCacheEntry},+ cache_manager::{CachedFileMetadataEntry, FileMetadataCache, FileMetadataCacheEntry},lru_queue::LruQueue,};/// Handles the inner state of the [`DefaultFilesMetadataCache`] struct.struct DefaultFilesMetadataCacheState {- lru_queue: LruQueue<Path, (ObjectMeta, Arc<dyn FileMetadata>)>,+ lru_queue: LruQueue<Path, CachedFileMetadataEntry>,memory_limit: usize,memory_used: usize,cache_hits: HashMap<Path, usize>,@@ -46,35 +43,18 @@ impl DefaultFilesMetadataCacheState {}}- /// Returns the respective entry from the cache, if it exists and the `size` and `last_modified`- /// properties from [`ObjectMeta`] match.+ /// Returns the respective entry from the cache, if it exists./// If the entry exists, it becomes the most recently used.- fn get(&mut self, k: &ObjectMeta) -> Option<Arc<dyn FileMetadata>> {- self.lru_queue- .get(&k.location)- .map(|(object_meta, metadata)| {- if object_meta.size != k.size- || object_meta.last_modified != k.last_modified- {- None- } else {- *self.cache_hits.entry(k.location.clone()).or_insert(0) += 1;- Some(Arc::clone(metadata))- }- })- .unwrap_or(None)+ fn get(&mut self, k: &Path) -> Option<CachedFileMetadataEntry> {+ self.lru_queue.get(k).cloned().inspect(|_| {+ *self.cache_hits.entry(k.clone()).or_insert(0) += 1;+ })}- /// Checks if the metadata is currently cached (entry exists and the `size` and `last_modified`- /// properties of [`ObjectMeta`] match).+ /// Checks if the metadata is currently cached./// The LRU queue is not updated.- fn contains_key(&self, k: &ObjectMeta) -> bool {- self.lru_queue- .peek(&k.location)- .map(|(object_meta, _)| {- object_meta.size == k.size && object_meta.last_modified == k.last_modified- })- .unwrap_or(false)+ fn contains_key(&self, k: &Path) -> bool {+ self.lru_queue.peek(k).is_some()}/// Adds a new key-value pair to cache, meaning LRU entries might be evicted if required.@@ -82,35 +62,34 @@ impl DefaultFilesMetadataCacheState {/// If the size of the metadata is greater than the `memory_limit`, the value is not inserted.fn put(&mut self,- key: ObjectMeta,- value: Arc<dyn FileMetadata>,- ) -> Option<Arc<dyn FileMetadata>> {- let value_size = value.memory_size();+ key: Path,+ value: CachedFileMetadataEntry,+ ) -> Option<CachedFileMetadataEntry> {+ let value_size = value.file_metadata.memory_size();// no point in trying to add this value to the cache if it cannot fit entirelyif value_size > self.memory_limit {return None;}- self.cache_hits.insert(key.location.clone(), 0);+ self.cache_hits.insert(key.clone(), 0);// if the key is already in the cache, the old value is removed- let old_value = self.lru_queue.put(key.location.clone(), (key, value));+ let old_value = self.lru_queue.put(key, value);self.memory_used += value_size;- if let Some((_, ref old_metadata)) = old_value {- self.memory_used -= old_metadata.memory_size();+ if let Some(ref old_entry) = old_value {+ self.memory_used -= old_entry.file_metadata.memory_size();}self.evict_entries();- old_value.map(|v| v.1)+ old_value}/// Evicts entries from the LRU cache until `memory_used` is lower than `memory_limit`.fn evict_entries(&mut self) {while self.memory_used > self.memory_limit {if let Some(removed) = self.lru_queue.pop() {- let metadata: Arc<dyn FileMetadata> = removed.1.1;- self.memory_used -= metadata.memory_size();+ self.memory_used -= removed.1.file_metadata.memory_size();} else {// cache is empty while memory_used > memory_limit, cannot happendebug_assert!(@@ -123,11 +102,11 @@ impl DefaultFilesMetadataCacheState {}/// Removes an entry from the cache and returns it, if it exists.- fn remove(&mut self, k: &ObjectMeta) -> Option<Arc<dyn FileMetadata>> {- if let Some((_, old_metadata)) = self.lru_queue.remove(&k.location) {- self.memory_used -= old_metadata.memory_size();- self.cache_hits.remove(&k.location);- Some(old_metadata)+ fn remove(&mut self, k: &Path) -> Option<CachedFileMetadataEntry> {+ if let Some(old_entry) = self.lru_queue.remove(k) {+ self.memory_used -= old_entry.file_metadata.memory_size();+ self.cache_hits.remove(k);+ Some(old_entry)} else {None}@@ -150,8 +129,8 @@ impl DefaultFilesMetadataCacheState {////// Collected file embedded metadata cache.///-/// The metadata for each file is invalidated when the file size or last-/// modification time have been changed.+/// The metadata for each file is validated by comparing the cached [`ObjectMeta`]+/// (size and last_modified) against the current file state using `cached.is_valid_for(¤t_meta)`.////// # Internal details///@@ -160,11 +139,7 @@ impl DefaultFilesMetadataCacheState {/// size of the cached entries exceeds `memory_limit`, the least recently used entries/// are evicted until the total size is lower than `memory_limit`.///-/// # `Extra` Handling-///-/// Users should use the [`Self::get`] and [`Self::put`] methods. The-/// [`Self::get_with_extra`] and [`Self::put_with_extra`] methods simply call-/// `get` and `put`, respectively.+/// [`ObjectMeta`]: object_store::ObjectMetapub struct DefaultFilesMetadataCache {// the state is wrapped in a Mutex to ensure the operations are atomicstate: Mutex<DefaultFilesMetadataCacheState>,@@ -189,78 +164,27 @@ impl DefaultFilesMetadataCache {}}-impl FileMetadataCache for DefaultFilesMetadataCache {- fn cache_limit(&self) -> usize {- let state = self.state.lock().unwrap();- state.memory_limit- }-- fn update_cache_limit(&self, limit: usize) {- let mut state = self.state.lock().unwrap();- state.memory_limit = limit;- state.evict_entries();- }-- fn list_entries(&self) -> HashMap<Path, FileMetadataCacheEntry> {- let state = self.state.lock().unwrap();- let mut entries = HashMap::<Path, FileMetadataCacheEntry>::new();-- for (path, (object_meta, metadata)) in state.lru_queue.list_entries() {- entries.insert(- path.clone(),- FileMetadataCacheEntry {- object_meta: object_meta.clone(),- size_bytes: metadata.memory_size(),- hits: *state.cache_hits.get(path).expect("entry must exist"),- extra: metadata.extra_info(),- },- );- }-- entries- }-}--impl CacheAccessor<ObjectMeta, Arc<dyn FileMetadata>> for DefaultFilesMetadataCache {- type Extra = ObjectMeta;-- fn get(&self, k: &ObjectMeta) -> Option<Arc<dyn FileMetadata>> {+impl CacheAccessor<Path, CachedFileMetadataEntry> for DefaultFilesMetadataCache {+ fn get(&self, key: &Path) -> Option<CachedFileMetadataEntry> {let mut state = self.state.lock().unwrap();- state.get(k)- }-- fn get_with_extra(- &self,- k: &ObjectMeta,- _e: &Self::Extra,- ) -> Option<Arc<dyn FileMetadata>> {- self.get(k)+ state.get(key)}fn put(&self,- key: &ObjectMeta,- value: Arc<dyn FileMetadata>,- ) -> Option<Arc<dyn FileMetadata>> {+ key: &Path,+ value: CachedFileMetadataEntry,+ ) -> Option<CachedFileMetadataEntry> {let mut state = self.state.lock().unwrap();state.put(key.clone(), value)}- fn put_with_extra(- &self,- key: &ObjectMeta,- value: Arc<dyn FileMetadata>,- _e: &Self::Extra,- ) -> Option<Arc<dyn FileMetadata>> {- self.put(key, value)- }-- fn remove(&self, k: &ObjectMeta) -> Option<Arc<dyn FileMetadata>> {+ fn remove(&self, k: &Path) -> Option<CachedFileMetadataEntry> {let mut state = self.state.lock().unwrap();state.remove(k)}- fn contains_key(&self, k: &ObjectMeta) -> bool {+ fn contains_key(&self, k: &Path) -> bool {let state = self.state.lock().unwrap();state.contains_key(k)}@@ -280,6 +204,38 @@ impl CacheAccessor<ObjectMeta, Arc<dyn FileMetadata>> for DefaultFilesMetadataCa}}+impl FileMetadataCache for DefaultFilesMetadataCache {+ fn cache_limit(&self) -> usize {+ let state = self.state.lock().unwrap();+ state.memory_limit+ }++ fn update_cache_limit(&self, limit: usize) {+ let mut state = self.state.lock().unwrap();+ state.memory_limit = limit;+ state.evict_entries();+ }++ fn list_entries(&self) -> HashMap<Path, FileMetadataCacheEntry> {+ let state = self.state.lock().unwrap();+ let mut entries = HashMap::<Path, FileMetadataCacheEntry>::new();++ for (path, entry) in state.lru_queue.list_entries() {+ entries.insert(+ path.clone(),+ FileMetadataCacheEntry {+ object_meta: entry.meta.clone(),+ size_bytes: entry.file_metadata.memory_size(),+ hits: *state.cache_hits.get(path).expect("entry must exist"),+ extra: entry.file_metadata.extra_info(),+ },+ );+ }++ entries+ }+}+#[cfg(test)]mod tests {use std::collections::HashMap;@@ -287,7 +243,7 @@ mod tests {use crate::cache::CacheAccessor;use crate::cache::cache_manager::{- FileMetadata, FileMetadataCache, FileMetadataCacheEntry,+ CachedFileMetadataEntry, FileMetadata, FileMetadataCache, FileMetadataCacheEntry,};use crate::cache::file_metadata_cache::DefaultFilesMetadataCache;use object_store::ObjectMeta;@@ -311,67 +267,77 @@ mod tests {}}- #[test]- fn test_default_file_metadata_cache() {- let object_meta = ObjectMeta {- location: Path::from("test"),+ fn create_test_object_meta(path: &str, size: usize) -> ObjectMeta {+ ObjectMeta {+ location: Path::from(path),last_modified: chrono::DateTime::parse_from_rfc3339("2025-07-29T12:12:12+00:00",).unwrap().into(),- size: 1024,+ size: size as u64,e_tag: None,version: None,- };+ }+ }++ #[test]+ fn test_default_file_metadata_cache() {+ let object_meta = create_test_object_meta("test", 1024);let metadata: Arc<dyn FileMetadata> = Arc::new(TestFileMetadata {metadata: "retrieved_metadata".to_owned(),});let cache = DefaultFilesMetadataCache::new(1024 * 1024);- assert!(cache.get(&object_meta).is_none());- // put- cache.put(&object_meta, Arc::clone(&metadata));+ // Cache miss+ assert!(cache.get(&object_meta.location).is_none());++ // Put a value+ let cached_entry =+ CachedFileMetadataEntry::new(object_meta.clone(), Arc::clone(&metadata));+ cache.put(&object_meta.location, cached_entry);- // get and contains of a valid entry- assert!(cache.contains_key(&object_meta));- let value = cache.get(&object_meta);- assert!(value.is_some());- let test_file_metadata = Arc::downcast::<TestFileMetadata>(value.unwrap());+ // Verify the cached value+ assert!(cache.contains_key(&object_meta.location));+ let result = cache.get(&object_meta.location).unwrap();+ let test_file_metadata = Arc::downcast::<TestFileMetadata>(result.file_metadata);assert!(test_file_metadata.is_ok());assert_eq!(test_file_metadata.unwrap().metadata, "retrieved_metadata");- // file size changed- let mut object_meta2 = object_meta.clone();- object_meta2.size = 2048;- assert!(cache.get(&object_meta2).is_none());- assert!(!cache.contains_key(&object_meta2));-- // file last_modified changed- let mut object_meta2 = object_meta.clone();- object_meta2.last_modified =- chrono::DateTime::parse_from_rfc3339("2025-07-29T13:13:13+00:00")- .unwrap()- .into();- assert!(cache.get(&object_meta2).is_none());- assert!(!cache.contains_key(&object_meta2));-- // different file- let mut object_meta2 = object_meta.clone();- object_meta2.location = Path::from("test2");- assert!(cache.get(&object_meta2).is_none());- assert!(!cache.contains_key(&object_meta2));+ // Cache hit - check validation+ let result2 = cache.get(&object_meta.location).unwrap();+ assert!(result2.is_valid_for(&object_meta));++ // File size changed - closure should detect invalidity+ let object_meta2 = create_test_object_meta("test", 2048);+ let result3 = cache.get(&object_meta2.location).unwrap();+ // Cached entry should NOT be valid for new meta+ assert!(!result3.is_valid_for(&object_meta2));++ // Return new entry+ let new_entry =+ CachedFileMetadataEntry::new(object_meta2.clone(), Arc::clone(&metadata));+ cache.put(&object_meta2.location, new_entry);++ let result4 = cache.get(&object_meta2.location).unwrap();+ assert_eq!(result4.meta.size, 2048);// remove- cache.remove(&object_meta);- assert!(cache.get(&object_meta).is_none());- assert!(!cache.contains_key(&object_meta));+ cache.remove(&object_meta.location);+ assert!(!cache.contains_key(&object_meta.location));// len and clear- cache.put(&object_meta, Arc::clone(&metadata));- cache.put(&object_meta2, metadata);+ let object_meta3 = create_test_object_meta("test3", 100);+ cache.put(+ &object_meta.location,+ CachedFileMetadataEntry::new(object_meta.clone(), Arc::clone(&metadata)),+ );+ cache.put(+ &object_meta3.location,+ CachedFileMetadataEntry::new(object_meta3.clone(), Arc::clone(&metadata)),+ );assert_eq!(cache.len(), 2);cache.clear();assert_eq!(cache.len(), 0);@@ -402,92 +368,129 @@ mod tests {let (object_meta2, metadata2) = generate_test_metadata_with_size("2", 500);let (object_meta3, metadata3) = generate_test_metadata_with_size("3", 300);- cache.put(&object_meta1, metadata1);- cache.put(&object_meta2, metadata2);- cache.put(&object_meta3, metadata3);+ cache.put(+ &object_meta1.location,+ CachedFileMetadataEntry::new(object_meta1.clone(), metadata1),+ );+ cache.put(+ &object_meta2.location,+ CachedFileMetadataEntry::new(object_meta2.clone(), metadata2),+ );+ cache.put(+ &object_meta3.location,+ CachedFileMetadataEntry::new(object_meta3.clone(), metadata3),+ );// all entries will fitassert_eq!(cache.len(), 3);assert_eq!(cache.memory_used(), 900);- assert!(cache.contains_key(&object_meta1));- assert!(cache.contains_key(&object_meta2));- assert!(cache.contains_key(&object_meta3));+ assert!(cache.contains_key(&object_meta1.location));+ assert!(cache.contains_key(&object_meta2.location));+ assert!(cache.contains_key(&object_meta3.location));// add a new entry which will remove the least recently used ("1")let (object_meta4, metadata4) = generate_test_metadata_with_size("4", 200);- cache.put(&object_meta4, metadata4);+ cache.put(+ &object_meta4.location,+ CachedFileMetadataEntry::new(object_meta4.clone(), metadata4),+ );assert_eq!(cache.len(), 3);assert_eq!(cache.memory_used(), 1000);- assert!(!cache.contains_key(&object_meta1));- assert!(cache.contains_key(&object_meta4));+ assert!(!cache.contains_key(&object_meta1.location));+ assert!(cache.contains_key(&object_meta4.location));// get entry "2", which will move it to the top of the queue, and add a new one which will// remove the new least recently used ("3")- cache.get(&object_meta2);+ let _ = cache.get(&object_meta2.location);let (object_meta5, metadata5) = generate_test_metadata_with_size("5", 100);- cache.put(&object_meta5, metadata5);+ cache.put(+ &object_meta5.location,+ CachedFileMetadataEntry::new(object_meta5.clone(), metadata5),+ );assert_eq!(cache.len(), 3);assert_eq!(cache.memory_used(), 800);- assert!(!cache.contains_key(&object_meta3));- assert!(cache.contains_key(&object_meta5));+ assert!(!cache.contains_key(&object_meta3.location));+ assert!(cache.contains_key(&object_meta5.location));// new entry which will not be able to fit in the 1000 bytes allocatedlet (object_meta6, metadata6) = generate_test_metadata_with_size("6", 1200);- cache.put(&object_meta6, metadata6);+ cache.put(+ &object_meta6.location,+ CachedFileMetadataEntry::new(object_meta6.clone(), metadata6),+ );assert_eq!(cache.len(), 3);assert_eq!(cache.memory_used(), 800);- assert!(!cache.contains_key(&object_meta6));+ assert!(!cache.contains_key(&object_meta6.location));// new entry which is able to fit without removing any entrylet (object_meta7, metadata7) = generate_test_metadata_with_size("7", 200);- cache.put(&object_meta7, metadata7);+ cache.put(+ &object_meta7.location,+ CachedFileMetadataEntry::new(object_meta7.clone(), metadata7),+ );assert_eq!(cache.len(), 4);assert_eq!(cache.memory_used(), 1000);- assert!(cache.contains_key(&object_meta7));+ assert!(cache.contains_key(&object_meta7.location));// new entry which will remove all other entrieslet (object_meta8, metadata8) = generate_test_metadata_with_size("8", 999);- cache.put(&object_meta8, metadata8);+ cache.put(+ &object_meta8.location,+ CachedFileMetadataEntry::new(object_meta8.clone(), metadata8),+ );assert_eq!(cache.len(), 1);assert_eq!(cache.memory_used(), 999);- assert!(cache.contains_key(&object_meta8));+ assert!(cache.contains_key(&object_meta8.location));// when updating an entry, the previous ones are not unnecessarily removedlet (object_meta9, metadata9) = generate_test_metadata_with_size("9", 300);let (object_meta10, metadata10) = generate_test_metadata_with_size("10", 200);let (object_meta11_v1, metadata11_v1) =generate_test_metadata_with_size("11", 400);- cache.put(&object_meta9, metadata9);- cache.put(&object_meta10, metadata10);- cache.put(&object_meta11_v1, metadata11_v1);+ cache.put(+ &object_meta9.location,+ CachedFileMetadataEntry::new(object_meta9.clone(), metadata9),+ );+ cache.put(+ &object_meta10.location,+ CachedFileMetadataEntry::new(object_meta10.clone(), metadata10),+ );+ cache.put(+ &object_meta11_v1.location,+ CachedFileMetadataEntry::new(object_meta11_v1.clone(), metadata11_v1),+ );assert_eq!(cache.memory_used(), 900);assert_eq!(cache.len(), 3);let (object_meta11_v2, metadata11_v2) =generate_test_metadata_with_size("11", 500);- cache.put(&object_meta11_v2, metadata11_v2);+ cache.put(+ &object_meta11_v2.location,+ CachedFileMetadataEntry::new(object_meta11_v2.clone(), metadata11_v2),+ );assert_eq!(cache.memory_used(), 1000);assert_eq!(cache.len(), 3);- assert!(cache.contains_key(&object_meta9));- assert!(cache.contains_key(&object_meta10));- assert!(cache.contains_key(&object_meta11_v2));- assert!(!cache.contains_key(&object_meta11_v1));+ assert!(cache.contains_key(&object_meta9.location));+ assert!(cache.contains_key(&object_meta10.location));+ assert!(cache.contains_key(&object_meta11_v2.location));// when updating an entry that now exceeds the limit, the LRU ("9") needs to be removedlet (object_meta11_v3, metadata11_v3) =generate_test_metadata_with_size("11", 501);- cache.put(&object_meta11_v3, metadata11_v3);+ cache.put(+ &object_meta11_v3.location,+ CachedFileMetadataEntry::new(object_meta11_v3.clone(), metadata11_v3),+ );assert_eq!(cache.memory_used(), 701);assert_eq!(cache.len(), 2);- assert!(cache.contains_key(&object_meta10));- assert!(cache.contains_key(&object_meta11_v3));- assert!(!cache.contains_key(&object_meta11_v2));+ assert!(cache.contains_key(&object_meta10.location));+ assert!(cache.contains_key(&object_meta11_v3.location));// manually removing an entry that is not the LRU- cache.remove(&object_meta11_v3);+ cache.remove(&object_meta11_v3.location);assert_eq!(cache.len(), 1);assert_eq!(cache.memory_used(), 200);- assert!(cache.contains_key(&object_meta10));- assert!(!cache.contains_key(&object_meta11_v3));+ assert!(cache.contains_key(&object_meta10.location));+ assert!(!cache.contains_key(&object_meta11_v3.location));// clearcache.clear();@@ -498,17 +501,26 @@ mod tests {let (object_meta12, metadata12) = generate_test_metadata_with_size("12", 300);let (object_meta13, metadata13) = generate_test_metadata_with_size("13", 200);let (object_meta14, metadata14) = generate_test_metadata_with_size("14", 500);- cache.put(&object_meta12, metadata12);- cache.put(&object_meta13, metadata13);- cache.put(&object_meta14, metadata14);+ cache.put(+ &object_meta12.location,+ CachedFileMetadataEntry::new(object_meta12.clone(), metadata12),+ );+ cache.put(+ &object_meta13.location,+ CachedFileMetadataEntry::new(object_meta13.clone(), metadata13),+ );+ cache.put(+ &object_meta14.location,+ CachedFileMetadataEntry::new(object_meta14.clone(), metadata14),+ );assert_eq!(cache.len(), 3);assert_eq!(cache.memory_used(), 1000);cache.update_cache_limit(600);assert_eq!(cache.len(), 1);assert_eq!(cache.memory_used(), 500);- assert!(!cache.contains_key(&object_meta12));- assert!(!cache.contains_key(&object_meta13));- assert!(cache.contains_key(&object_meta14));+ assert!(!cache.contains_key(&object_meta12.location));+ assert!(!cache.contains_key(&object_meta13.location));+ assert!(cache.contains_key(&object_meta14.location));}#[test]@@ -519,9 +531,18 @@ mod tests {let (object_meta3, metadata3) = generate_test_metadata_with_size("3", 300);// initial entries, all will have hits = 0- cache.put(&object_meta1, metadata1);- cache.put(&object_meta2, metadata2);- cache.put(&object_meta3, metadata3);+ cache.put(+ &object_meta1.location,+ CachedFileMetadataEntry::new(object_meta1.clone(), metadata1),+ );+ cache.put(+ &object_meta2.location,+ CachedFileMetadataEntry::new(object_meta2.clone(), metadata2),+ );+ cache.put(+ &object_meta3.location,+ CachedFileMetadataEntry::new(object_meta3.clone(), metadata3),+ );assert_eq!(cache.list_entries(),HashMap::from([@@ -565,7 +586,7 @@ mod tests {);// new hit on "1"- cache.get(&object_meta1);+ let _ = cache.get(&object_meta1.location);assert_eq!(cache.list_entries(),HashMap::from([@@ -610,7 +631,10 @@ mod tests {// new entry, will evict "2"let (object_meta4, metadata4) = generate_test_metadata_with_size("4", 600);- cache.put(&object_meta4, metadata4);+ cache.put(+ &object_meta4.location,+ CachedFileMetadataEntry::new(object_meta4.clone(), metadata4),+ );assert_eq!(cache.list_entries(),HashMap::from([@@ -655,7 +679,10 @@ mod tests {// replace entry "1"let (object_meta1_new, metadata1_new) = generate_test_metadata_with_size("1", 50);- cache.put(&object_meta1_new, metadata1_new);+ cache.put(+ &object_meta1_new.location,+ CachedFileMetadataEntry::new(object_meta1_new.clone(), metadata1_new),+ );assert_eq!(cache.list_entries(),HashMap::from([@@ -699,7 +726,7 @@ mod tests {);// remove entry "4"- cache.remove(&object_meta4);+ cache.remove(&object_meta4.location);assert_eq!(cache.list_entries(),HashMap::from([diff --git a/datafusion/execution/src/cache/list_files_cache.rs b/datafusion/execution/src/cache/list_files_cache.rsindex 858219e5b8..c86a03574e 100644--- a/datafusion/execution/src/cache/list_files_cache.rs+++ b/datafusion/execution/src/cache/list_files_cache.rs@@ -26,7 +26,11 @@ use datafusion_common::TableReference;use datafusion_common::instant::Instant;use object_store::{ObjectMeta, path::Path};-use crate::cache::{CacheAccessor, cache_manager::ListFilesCache, lru_queue::LruQueue};+use crate::cache::{+ CacheAccessor,+ cache_manager::{CachedFileList, ListFilesCache},+ lru_queue::LruQueue,+};pub trait TimeProvider: Send + Sync + 'static {fn now(&self) -> Instant;@@ -52,11 +56,10 @@ impl TimeProvider for SystemTimeProvider {/// the cache exceeds `memory_limit`, the least recently used entries are evicted until the total/// size is lower than the `memory_limit`.///-/// # `Extra` Handling+/// # Cache API///-/// Users should use the [`Self::get`] and [`Self::put`] methods. The-/// [`Self::get_with_extra`] and [`Self::put_with_extra`] methods simply call-/// `get` and `put`, respectively.+/// Uses `get` and `put` methods for cache operations. TTL validation is handled internally -+/// expired entries return `None` from `get`.pub struct DefaultListFilesCache {state: Mutex<DefaultListFilesCacheState>,time_provider: Arc<dyn TimeProvider>,@@ -86,43 +89,30 @@ impl DefaultListFilesCache {self.time_provider = provider;self}-- /// Returns the cache's memory limit in bytes.- pub fn cache_limit(&self) -> usize {- self.state.lock().unwrap().memory_limit- }-- /// Updates the cache with a new memory limit in bytes.- pub fn update_cache_limit(&self, limit: usize) {- let mut state = self.state.lock().unwrap();- state.memory_limit = limit;- state.evict_entries();- }-- /// Returns the TTL (time-to-live) applied to cache entries.- pub fn cache_ttl(&self) -> Option<Duration> {- self.state.lock().unwrap().ttl- }}#[derive(Clone, PartialEq, Debug)]pub struct ListFilesEntry {- pub metas: Arc<Vec<ObjectMeta>>,+ pub metas: CachedFileList,pub size_bytes: usize,pub expires: Option<Instant>,}impl ListFilesEntry {fn try_new(- metas: Arc<Vec<ObjectMeta>>,+ cached_file_list: CachedFileList,ttl: Option<Duration>,now: Instant,) -> Option<Self> {- let size_bytes = (metas.capacity() * size_of::<ObjectMeta>())- + metas.iter().map(meta_heap_bytes).reduce(|acc, b| acc + b)?;+ let size_bytes = (cached_file_list.files.capacity() * size_of::<ObjectMeta>())+ + cached_file_list+ .files+ .iter()+ .map(meta_heap_bytes)+ .reduce(|acc, b| acc + b)?;Some(Self {- metas,+ metas: cached_file_list,size_bytes,expires: ttl.map(|t| now + t),})@@ -184,66 +174,22 @@ impl DefaultListFilesCacheState {}}- /// Performs a prefix-aware cache lookup.+ /// Gets an entry from the cache, checking for expiration.///- /// # Arguments- /// * `table_base` - The table's base path (the cache key)- /// * `prefix` - Optional prefix filter relative to the table base path- /// * `now` - Current time for expiration checking- ///- /// # Behavior- /// - Fetches the cache entry for `table_base`- /// - If `prefix` is `Some`, filters results to only files matching `table_base/prefix`- /// - Returns the (potentially filtered) results- ///- /// # Example- /// ```text- /// get_with_prefix("my_table", Some("a=1"), now)- /// → Fetch cache entry for "my_table"- /// → Filter to files matching "my_table/a=1/*"- /// → Return filtered results- /// ```- fn get_with_prefix(- &mut self,- table_scoped_base_path: &TableScopedPath,- prefix: Option<&Path>,- now: Instant,- ) -> Option<Arc<Vec<ObjectMeta>>> {- let entry = self.lru_queue.get(table_scoped_base_path)?;+ /// Returns the cached file list if it exists and hasn't expired.+ /// If the entry has expired, it is removed from the cache.+ fn get(&mut self, key: &TableScopedPath, now: Instant) -> Option<CachedFileList> {+ let entry = self.lru_queue.get(key)?;// Check expirationif let Some(exp) = entry.expires&& now > exp{- self.remove(table_scoped_base_path);+ self.remove(key);return None;}- // Early return if no prefix filter - return all files- let Some(prefix) = prefix else {- return Some(Arc::clone(&entry.metas));- };-- // Build the full prefix path: table_base/prefix- let table_base = &table_scoped_base_path.path;- let mut parts: Vec<_> = table_base.parts().collect();- parts.extend(prefix.parts());- let full_prefix = Path::from_iter(parts);- let full_prefix_str = full_prefix.as_ref();-- // Filter files to only those matching the prefix- let filtered: Vec<ObjectMeta> = entry- .metas- .iter()- .filter(|meta| meta.location.as_ref().starts_with(full_prefix_str))- .cloned()- .collect();-- if filtered.is_empty() {- None- } else {- Some(Arc::new(filtered))- }+ Some(entry.metas.clone())}/// Checks if the respective entry is currently cached.@@ -273,9 +219,9 @@ impl DefaultListFilesCacheState {fn put(&mut self,key: &TableScopedPath,- value: Arc<Vec<ObjectMeta>>,+ value: CachedFileList,now: Instant,- ) -> Option<Arc<Vec<ObjectMeta>>> {+ ) -> Option<CachedFileList> {let entry = ListFilesEntry::try_new(value, self.ttl, now)?;let entry_size = entry.size_bytes;@@ -314,7 +260,7 @@ impl DefaultListFilesCacheState {}/// Removes an entry from the cache and returns it, if it exists.- fn remove(&mut self, k: &TableScopedPath) -> Option<Arc<Vec<ObjectMeta>>> {+ fn remove(&mut self, k: &TableScopedPath) -> Option<CachedFileList> {if let Some(entry) = self.lru_queue.remove(k) {self.memory_used -= entry.size_bytes;Some(entry.metas)@@ -335,6 +281,49 @@ impl DefaultListFilesCacheState {}}+impl CacheAccessor<TableScopedPath, CachedFileList> for DefaultListFilesCache {+ fn get(&self, key: &TableScopedPath) -> Option<CachedFileList> {+ let mut state = self.state.lock().unwrap();+ let now = self.time_provider.now();+ state.get(key, now)+ }++ fn put(+ &self,+ key: &TableScopedPath,+ value: CachedFileList,+ ) -> Option<CachedFileList> {+ let mut state = self.state.lock().unwrap();+ let now = self.time_provider.now();+ state.put(key, value, now)+ }++ fn remove(&self, k: &TableScopedPath) -> Option<CachedFileList> {+ let mut state = self.state.lock().unwrap();+ state.remove(k)+ }++ fn contains_key(&self, k: &TableScopedPath) -> bool {+ let mut state = self.state.lock().unwrap();+ let now = self.time_provider.now();+ state.contains_key(k, now)+ }++ fn len(&self) -> usize {+ let state = self.state.lock().unwrap();+ state.len()+ }++ fn clear(&self) {+ let mut state = self.state.lock().unwrap();+ state.clear();+ }++ fn name(&self) -> String {+ String::from("DefaultListFilesCache")+ }+}+impl ListFilesCache for DefaultListFilesCache {fn cache_limit(&self) -> usize {let state = self.state.lock().unwrap();@@ -385,88 +374,11 @@ impl ListFilesCache for DefaultListFilesCache {}}-impl CacheAccessor<TableScopedPath, Arc<Vec<ObjectMeta>>> for DefaultListFilesCache {- type Extra = Option<Path>;-- /// Gets all files for the given table base path.- ///- /// This is equivalent to calling `get_with_extra(k, &None)`.- fn get(&self, k: &TableScopedPath) -> Option<Arc<Vec<ObjectMeta>>> {- self.get_with_extra(k, &None)- }-- /// Performs a prefix-aware cache lookup.- ///- /// # Arguments- /// * `table_base` - The table's base path (the cache key)- /// * `prefix` - Optional prefix filter (relative to table base) for partition filtering- ///- /// # Behavior- /// - Fetches the cache entry for `table_base`- /// - If `prefix` is `Some`, filters results to only files matching `table_base/prefix`- /// - Returns the (potentially filtered) results- ///- /// This enables efficient partition pruning - a single cached listing of the full table- /// can serve queries for any partition subset without additional storage calls.- fn get_with_extra(- &self,- table_scoped_path: &TableScopedPath,- prefix: &Self::Extra,- ) -> Option<Arc<Vec<ObjectMeta>>> {- let mut state = self.state.lock().unwrap();- let now = self.time_provider.now();- state.get_with_prefix(table_scoped_path, prefix.as_ref(), now)- }-- fn put(- &self,- key: &TableScopedPath,- value: Arc<Vec<ObjectMeta>>,- ) -> Option<Arc<Vec<ObjectMeta>>> {- let mut state = self.state.lock().unwrap();- let now = self.time_provider.now();- state.put(key, value, now)- }-- fn put_with_extra(- &self,- key: &TableScopedPath,- value: Arc<Vec<ObjectMeta>>,- _e: &Self::Extra,- ) -> Option<Arc<Vec<ObjectMeta>>> {- self.put(key, value)- }-- fn remove(&self, k: &TableScopedPath) -> Option<Arc<Vec<ObjectMeta>>> {- let mut state = self.state.lock().unwrap();- state.remove(k)- }-- fn contains_key(&self, k: &TableScopedPath) -> bool {- let mut state = self.state.lock().unwrap();- let now = self.time_provider.now();- state.contains_key(k, now)- }-- fn len(&self) -> usize {- let state = self.state.lock().unwrap();- state.len()- }-- fn clear(&self) {- let mut state = self.state.lock().unwrap();- state.clear();- }-- fn name(&self) -> String {- String::from("DefaultListFilesCache")- }-}-#[cfg(test)]mod tests {use super::*;use chrono::DateTime;+ use std::thread;struct MockTimeProvider {base: Instant,@@ -513,22 +425,21 @@ mod tests {}}- /// Helper function to create a vector of ObjectMeta with at least meta_size bytes+ /// Helper function to create a CachedFileList with at least meta_size bytesfn create_test_list_files_entry(path: &str,count: usize,meta_size: usize,- ) -> (Path, Arc<Vec<ObjectMeta>>, usize) {+ ) -> (Path, CachedFileList, usize) {let metas: Vec<ObjectMeta> = (0..count).map(|i| create_test_object_meta(&format!("file{i}"), meta_size)).collect();- let metas = Arc::new(metas);// Calculate actual size using the same logic as ListFilesEntry::try_newlet size = (metas.capacity() * size_of::<ObjectMeta>())+ metas.iter().map(meta_heap_bytes).sum::<usize>();- (Path::from(path), metas, size)+ (Path::from(path), CachedFileList::new(metas), size)}#[test]@@ -542,25 +453,25 @@ mod tests {};// Initially cache is empty- assert!(cache.get(&key).is_none());assert!(!cache.contains_key(&key));assert_eq!(cache.len(), 0);- // Put an entry+ // Cache miss - get returns None+ assert!(cache.get(&key).is_none());++ // Put a valuelet meta = create_test_object_meta("file1", 50);- let value = Arc::new(vec![meta.clone()]);- cache.put(&key, Arc::clone(&value));+ cache.put(&key, CachedFileList::new(vec![meta]));- // Entry should be retrievable+ // Entry should be cachedassert!(cache.contains_key(&key));assert_eq!(cache.len(), 1);- let retrieved = cache.get(&key).unwrap();- assert_eq!(retrieved.len(), 1);- assert_eq!(retrieved[0].location, meta.location);+ let result = cache.get(&key).unwrap();+ assert_eq!(result.files.len(), 1);// Remove the entrylet removed = cache.remove(&key).unwrap();- assert_eq!(removed.len(), 1);+ assert_eq!(removed.files.len(), 1);assert!(!cache.contains_key(&key));assert_eq!(cache.len(), 0);@@ -575,8 +486,8 @@ mod tests {table: table_ref,path: path2,};- cache.put(&key1, Arc::clone(&value1));- cache.put(&key2, Arc::clone(&value2));+ cache.put(&key1, value1.clone());+ cache.put(&key2, value2.clone());assert_eq!(cache.len(), 2);// List cache entries@@ -686,7 +597,7 @@ mod tests {// Access path1 to move it to front (MRU)// Order is now: path2 (LRU), path3, path1 (MRU)- cache.get(&key1);+ let _ = cache.get(&key1);// Adding a new entry should evict path2 (the LRU)let (path4, value4, _) = create_test_list_files_entry("path4", 1, 100);@@ -725,6 +636,7 @@ mod tests {assert_eq!(cache.len(), 2);// Try to add an entry that's too large to fit in the cache+ // The entry is not stored (too large)let (path_large, value_large, _) = create_test_list_files_entry("large", 1, 1000);let key_large = TableScopedPath {table: table_ref,@@ -843,7 +755,7 @@ mod tests {};// Add three entriescache.put(&key1, value1);- cache.put(&key2, Arc::clone(&value2));+ cache.put(&key2, value2.clone());cache.put(&key3, value3_v1);assert_eq!(cache.len(), 3);@@ -858,7 +770,7 @@ mod tests {// Update path3 with larger size that requires evicting path1 (LRU)let (_, value3_v3, size3_v3) = create_test_list_files_entry("path3", 1, 200);- cache.put(&key3, Arc::clone(&value3_v3));+ cache.put(&key3, value3_v3.clone());assert_eq!(cache.len(), 2);assert!(!cache.contains_key(&key1)); // Evicted (was LRU)@@ -909,8 +821,8 @@ mod tests {table: table_ref,path: path2,};- cache.put(&key1, Arc::clone(&value1));- cache.put(&key2, Arc::clone(&value2));+ cache.put(&key1, value1.clone());+ cache.put(&key2, value2.clone());// Entries should be accessible immediatelyassert!(cache.get(&key1).is_some());@@ -940,11 +852,11 @@ mod tests {// Wait for TTL to expiremock_time.inc(Duration::from_millis(150));- // Entries should now return None and be removed when observed through get or contains_key- assert!(cache.get(&key1).is_none());- assert_eq!(cache.len(), 1); // path1 was removed by get()+ // Entries should now return None when observed through contains_key+ assert!(!cache.contains_key(&key1));+ assert_eq!(cache.len(), 1); // key1 was removed by contains_key()assert!(!cache.contains_key(&key2));- assert_eq!(cache.len(), 0); // path2 was removed by contains_key()+ assert_eq!(cache.len(), 0); // key2 was removed by contains_key()}#[test]@@ -986,7 +898,35 @@ mod tests {mock_time.inc(Duration::from_millis(151));assert!(!cache.contains_key(&key2)); // Expired- assert!(cache.contains_key(&key3)); // Still valid+ assert!(cache.contains_key(&key3)); // Still valid+ }++ #[test]+ fn test_ttl_expiration_in_get() {+ let ttl = Duration::from_millis(100);+ let cache = DefaultListFilesCache::new(10000, Some(ttl));++ let (path, value, _) = create_test_list_files_entry("path", 2, 50);+ let table_ref = Some(TableReference::from("table"));+ let key = TableScopedPath {+ table: table_ref,+ path,+ };++ // Cache the entry+ cache.put(&key, value.clone());++ // Entry should be accessible immediately+ let result = cache.get(&key);+ assert!(result.is_some());+ assert_eq!(result.unwrap().files.len(), 2);++ // Wait for TTL to expire+ thread::sleep(Duration::from_millis(150));++ // Get should return None because entry expired+ let result2 = cache.get(&key);+ assert!(result2.is_none());}#[test]@@ -1035,28 +975,28 @@ mod tests {#[test]fn test_entry_creation() {// Test with empty vector- let empty_vec: Arc<Vec<ObjectMeta>> = Arc::new(vec![]);+ let empty_list = CachedFileList::new(vec![]);let now = Instant::now();- let entry = ListFilesEntry::try_new(empty_vec, None, now);+ let entry = ListFilesEntry::try_new(empty_list, None, now);assert!(entry.is_none());// Validate entry sizelet metas: Vec<ObjectMeta> = (0..5).map(|i| create_test_object_meta(&format!("file{i}"), 30)).collect();- let metas = Arc::new(metas);- let entry = ListFilesEntry::try_new(metas, None, now).unwrap();- assert_eq!(entry.metas.len(), 5);+ let cached_list = CachedFileList::new(metas);+ let entry = ListFilesEntry::try_new(cached_list, None, now).unwrap();+ assert_eq!(entry.metas.files.len(), 5);// Size should be: capacity * sizeof(ObjectMeta) + (5 * 30) for heap bytes- let expected_size =- (entry.metas.capacity() * size_of::<ObjectMeta>()) + (entry.metas.len() * 30);+ let expected_size = (entry.metas.files.capacity() * size_of::<ObjectMeta>())+ + (entry.metas.files.len() * 30);assert_eq!(entry.size_bytes, expected_size);// Test with TTLlet meta = create_test_object_meta("file", 50);let ttl = Duration::from_secs(10);- let entry =- ListFilesEntry::try_new(Arc::new(vec![meta]), Some(ttl), now).unwrap();+ let cached_list = CachedFileList::new(vec![meta]);+ let entry = ListFilesEntry::try_new(cached_list, Some(ttl), now).unwrap();assert!(entry.expires.unwrap() > now);}@@ -1110,7 +1050,7 @@ mod tests {}}- // Prefix-aware cache tests+ // Prefix filtering tests using CachedFileList::filter_by_prefix/// Helper function to create ObjectMeta with a specific location pathfn create_object_meta_with_path(location: &str) -> ObjectMeta {@@ -1126,18 +1066,17 @@ mod tests {}#[test]- fn test_prefix_aware_cache_hit() {- // Scenario: Cache has full table listing, query for partition returns filtered results+ fn test_prefix_filtering() {let cache = DefaultListFilesCache::new(100000, None);// Create files for a partitioned tablelet table_base = Path::from("my_table");- let files = Arc::new(vec![+ let files = vec![create_object_meta_with_path("my_table/a=1/file1.parquet"),create_object_meta_with_path("my_table/a=1/file2.parquet"),create_object_meta_with_path("my_table/a=2/file3.parquet"),create_object_meta_with_path("my_table/a=2/file4.parquet"),- ]);+ ];// Cache the full table listinglet table_ref = Some(TableReference::from("table"));@@ -1145,16 +1084,13 @@ mod tests {table: table_ref,path: table_base,};- cache.put(&key, files);+ cache.put(&key, CachedFileList::new(files));- // Query for partition a=1 using get_with_extra- // New API: get_with_extra(table_base, Some(relative_prefix))- let prefix_a1 = Some(Path::from("a=1"));- let result = cache.get_with_extra(&key, &prefix_a1);+ let result = cache.get(&key).unwrap();- // Should return filtered results (only files from a=1)- assert!(result.is_some());- let filtered = result.unwrap();+ // Filter for partition a=1+ let prefix_a1 = Some(Path::from("my_table/a=1"));+ let filtered = result.files_matching_prefix(&prefix_a1);assert_eq!(filtered.len(), 2);assert!(filtered@@ -1162,107 +1098,51 @@ mod tests {.all(|m| m.location.as_ref().starts_with("my_table/a=1")));- // Query for partition a=2- let prefix_a2 = Some(Path::from("a=2"));- let result_2 = cache.get_with_extra(&key, &prefix_a2);-- assert!(result_2.is_some());- let filtered_2 = result_2.unwrap();+ // Filter for partition a=2+ let prefix_a2 = Some(Path::from("my_table/a=2"));+ let filtered_2 = result.files_matching_prefix(&prefix_a2);assert_eq!(filtered_2.len(), 2);assert!(filtered_2.iter().all(|m| m.location.as_ref().starts_with("my_table/a=2")));- }-- #[test]- fn test_prefix_aware_cache_no_filter_returns_all() {- // Scenario: Query with no prefix filter should return all files- let cache = DefaultListFilesCache::new(100000, None);-- let table_base = Path::from("my_table");-- // Cache full table listing with 4 files- let full_files = Arc::new(vec![- create_object_meta_with_path("my_table/a=1/file1.parquet"),- create_object_meta_with_path("my_table/a=1/file2.parquet"),- create_object_meta_with_path("my_table/a=2/file3.parquet"),- create_object_meta_with_path("my_table/a=2/file4.parquet"),- ]);- let table_ref = Some(TableReference::from("table"));- let key = TableScopedPath {- table: table_ref,- path: table_base,- };- cache.put(&key, full_files);- // Query with no prefix filter (None) should return all 4 files- let result = cache.get_with_extra(&key, &None);- assert!(result.is_some());- let files = result.unwrap();- assert_eq!(files.len(), 4);-- // Also test using get() which delegates to get_with_extra(&None)- let result_get = cache.get(&key);- assert!(result_get.is_some());- assert_eq!(result_get.unwrap().len(), 4);+ // No filter returns all+ let all = result.files_matching_prefix(&None);+ assert_eq!(all.len(), 4);}#[test]- fn test_prefix_aware_cache_miss_no_entry() {- // Scenario: Table not cached, query should miss+ fn test_prefix_no_matching_files() {let cache = DefaultListFilesCache::new(100000, None);let table_base = Path::from("my_table");- let table_ref = Some(TableReference::from("table"));- let key = TableScopedPath {- table: table_ref,- path: table_base,- };-- // Query for full table should miss (nothing cached)- let result = cache.get_with_extra(&key, &None);- assert!(result.is_none());-- // Query with prefix should also miss- let prefix = Some(Path::from("a=1"));- let result_2 = cache.get_with_extra(&key, &prefix);- assert!(result_2.is_none());- }-- #[test]- fn test_prefix_aware_cache_no_matching_files() {- // Scenario: Cache has table listing but no files match the requested partition- let cache = DefaultListFilesCache::new(100000, None);-- let table_base = Path::from("my_table");- let files = Arc::new(vec![+ let files = vec![create_object_meta_with_path("my_table/a=1/file1.parquet"),create_object_meta_with_path("my_table/a=2/file2.parquet"),- ]);+ ];+let table_ref = Some(TableReference::from("table"));let key = TableScopedPath {table: table_ref,path: table_base,};- cache.put(&key, files);+ cache.put(&key, CachedFileList::new(files));+ let result = cache.get(&key).unwrap();// Query for partition a=3 which doesn't exist- let prefix_a3 = Some(Path::from("a=3"));- let result = cache.get_with_extra(&key, &prefix_a3);-- // Should return None since no files match- assert!(result.is_none());+ let prefix_a3 = Some(Path::from("my_table/a=3"));+ let filtered = result.files_matching_prefix(&prefix_a3);+ assert!(filtered.is_empty());}#[test]- fn test_prefix_aware_nested_partitions() {- // Scenario: Table with multiple partition levels (e.g., year/month/day)+ fn test_nested_partitions() {let cache = DefaultListFilesCache::new(100000, None);let table_base = Path::from("events");- let files = Arc::new(vec![+ let files = vec![create_object_meta_with_path("events/year=2024/month=01/day=01/file1.parquet",),@@ -1275,72 +1155,25 @@ mod tests {create_object_meta_with_path("events/year=2025/month=01/day=01/file4.parquet",),- ]);+ ];+let table_ref = Some(TableReference::from("table"));let key = TableScopedPath {table: table_ref,path: table_base,};- cache.put(&key, files);-- // Query for year=2024/month=01 (should get 2 files)- let prefix_month = Some(Path::from("year=2024/month=01"));- let result = cache.get_with_extra(&key, &prefix_month);- assert!(result.is_some());- assert_eq!(result.unwrap().len(), 2);-- // Query for year=2024 (should get 3 files)- let prefix_year = Some(Path::from("year=2024"));- let result_year = cache.get_with_extra(&key, &prefix_year);- assert!(result_year.is_some());- assert_eq!(result_year.unwrap().len(), 3);-- // Query for specific day (should get 1 file)- let prefix_day = Some(Path::from("year=2024/month=01/day=01"));- let result_day = cache.get_with_extra(&key, &prefix_day);- assert!(result_day.is_some());- assert_eq!(result_day.unwrap().len(), 1);- }+ cache.put(&key, CachedFileList::new(files));+ let result = cache.get(&key).unwrap();- #[test]- fn test_prefix_aware_different_tables() {- // Scenario: Multiple tables cached, queries should not cross-contaminate- let cache = DefaultListFilesCache::new(100000, None);+ // Filter for year=2024/month=01+ let prefix_month = Some(Path::from("events/year=2024/month=01"));+ let filtered = result.files_matching_prefix(&prefix_month);+ assert_eq!(filtered.len(), 2);- let table_a = Path::from("table_a");- let table_b = Path::from("table_b");-- let files_a = Arc::new(vec![create_object_meta_with_path(- "table_a/part=1/file1.parquet",- )]);- let files_b = Arc::new(vec![- create_object_meta_with_path("table_b/part=1/file1.parquet"),- create_object_meta_with_path("table_b/part=2/file2.parquet"),- ]);-- let table_ref_a = Some(TableReference::from("table_a"));- let table_ref_b = Some(TableReference::from("table_b"));- let key_a = TableScopedPath {- table: table_ref_a,- path: table_a,- };- let key_b = TableScopedPath {- table: table_ref_b,- path: table_b,- };- cache.put(&key_a, files_a);- cache.put(&key_b, files_b);-- // Query table_a should only return table_a files- let result_a = cache.get(&key_a);- assert!(result_a.is_some());- assert_eq!(result_a.unwrap().len(), 1);-- // Query table_b with prefix should only return matching table_b files- let prefix = Some(Path::from("part=1"));- let result_b = cache.get_with_extra(&key_b, &prefix);- assert!(result_b.is_some());- assert_eq!(result_b.unwrap().len(), 1);+ // Filter for year=2024+ let prefix_year = Some(Path::from("events/year=2024"));+ let filtered_year = result.files_matching_prefix(&prefix_year);+ assert_eq!(filtered_year.len(), 3);}#[test]diff --git a/datafusion/execution/src/cache/mod.rs b/datafusion/execution/src/cache/mod.rsindex 93b9f0520b..417cb86cd9 100644--- a/datafusion/execution/src/cache/mod.rs+++ b/datafusion/execution/src/cache/mod.rs@@ -26,35 +26,54 @@ pub use file_metadata_cache::DefaultFilesMetadataCache;pub use list_files_cache::DefaultListFilesCache;pub use list_files_cache::TableScopedPath;-/// A trait that can be implemented to provide custom cache behavior for the caches managed by-/// [`cache_manager::CacheManager`].+/// Base trait for cache implementations with common operations.+///+/// This trait provides the fundamental cache operations (`get`, `put`, `remove`, etc.)+/// that all cache types share. Specific cache traits like [`cache_manager::FileStatisticsCache`],+/// [`cache_manager::ListFilesCache`], and [`cache_manager::FileMetadataCache`] extend this+/// trait with their specialized methods.+///+/// ## Thread Safety////// Implementations must handle their own locking via internal mutability, as methods do not/// take mutable references and may be accessed by multiple concurrent queries.+///+/// ## Validation Pattern+///+/// Validation metadata (e.g., file size, last modified time) should be embedded in the+/// value type `V`. The typical usage pattern is:+/// 1. Call `get(key)` to check for cached value+/// 2. If `Some(cached)`, validate with `cached.is_valid_for(¤t_meta)`+/// 3. If invalid or missing, compute new value and call `put(key, new_value)`pub trait CacheAccessor<K, V>: Send + Sync {- // Extra info but not part of the cache key or cache value.- type Extra: Clone;-- /// Get value from cache.- fn get(&self, k: &K) -> Option<V>;- /// Get value from cache.- fn get_with_extra(&self, k: &K, e: &Self::Extra) -> Option<V>;- /// Put value into cache. Returns the old value associated with the key if there was one.+ /// Get a cached entry if it exists.+ ///+ /// Returns the cached value without any validation. The caller should+ /// validate the returned value if freshness matters.+ fn get(&self, key: &K) -> Option<V>;++ /// Store a value in the cache.+ ///+ /// Returns the previous value if one existed.fn put(&self, key: &K, value: V) -> Option<V>;- /// Put value into cache. Returns the old value associated with the key if there was one.- fn put_with_extra(&self, key: &K, value: V, e: &Self::Extra) -> Option<V>;- /// Remove an entry from the cache, returning value if they existed in the map.++ /// Remove an entry from the cache, returning the value if it existed.fn remove(&self, k: &K) -> Option<V>;+/// Check if the cache contains a specific key.fn contains_key(&self, k: &K) -> bool;+/// Fetch the total number of cache entries.fn len(&self) -> usize;- /// Check if the Cache collection is empty or not.++ /// Check if the cache collection is empty.fn is_empty(&self) -> bool {self.len() == 0}+/// Remove all entries from the cache.fn clear(&self);+/// Return the cache name.fn name(&self) -> String;}