Apache DataFusion SQL Query Engine
test: Move default cache tests to default cache file (#23040)
Michael Kleena4c4b4d98650
diff --git a/datafusion/execution/src/cache/default_cache.rs b/datafusion/execution/src/cache/default_cache.rsindex ed27c80d86..a1d89619eb 100644--- a/datafusion/execution/src/cache/default_cache.rs+++ b/datafusion/execution/src/cache/default_cache.rs@@ -294,3 +294,1690 @@ impl<K: CacheKey, V: CacheValue> Cache<K, V> for DefaultCache<K, V> {.collect()}}++#[cfg(test)]+mod tests {+ use std::sync::Arc;++ use crate::cache::TableScopedPath;+ use crate::cache::cache_manager::{+ CachedFileList, DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, meta_heap_bytes,+ };+ use crate::cache::cache_manager::{+ CachedFileMetadata, DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,+ };+ use crate::cache::cache_manager::{CachedFileMetadataEntry, FileMetadata};+ use crate::cache::default_cache::DefaultCache;+ use crate::cache::default_cache::TimeProvider;+ use crate::cache::{Cache, CacheEntryInfo};+ use crate::cache::{CacheKey, CacheValue};+ use arrow::array::{Int32Array, ListArray, RecordBatch};+ use arrow::buffer::{OffsetBuffer, ScalarBuffer};+ use arrow::datatypes::{DataType, Field, Schema, TimeUnit};+ use chrono::DateTime;+ use datafusion_common::HashMap;+ use datafusion_common::TableReference;+ use datafusion_common::heap_size::{DFHeapSize, DFHeapSizeCtx};+ use datafusion_common::instant::Instant;+ use datafusion_common::stats::Precision;+ use datafusion_common::{ColumnStatistics, ScalarValue, Statistics};+ use datafusion_expr::ColumnarValue;+ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+ use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+ use object_store::ObjectMeta;+ use object_store::path::Path;+ use std::sync::Mutex;+ use std::thread;+ use std::time::Duration;++ pub struct TestFileMetadata {+ metadata: String,+ }++ impl FileMetadata for TestFileMetadata {+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }++ fn memory_size(&self) -> usize {+ self.metadata.len()+ }++ fn extra_info(&self) -> HashMap<String, String> {+ HashMap::from([("extra_info".to_owned(), "abc".to_owned())])+ }+ }++ impl PartialEq for CachedFileMetadataEntry {+ fn eq(&self, other: &Self) -> bool {+ self.meta == other.meta+ }+ }++ fn create_test_object_meta(path: &str, size: usize) -> ObjectMeta {+ ObjectMeta {+ location: Path::from(path),+ last_modified: DateTime::parse_from_rfc3339("2025-07-29T12:12:12+00:00")+ .unwrap()+ .into(),+ size: size as u64,+ e_tag: None,+ version: None,+ }+ }++ #[test]+ fn test_default_file_metadata_cache() {+ let object_meta = create_test_object_meta("test", 1024);++ let metadata: Arc<dyn FileMetadata> = Arc::new(TestFileMetadata {+ metadata: "retrieved_metadata".to_owned(),+ });++ let cache = DefaultCache::new(1024 * 1024);++ // Cache miss+ assert!(cache.get(&object_meta.location).is_none());++ // Put a value+ let cached_entry =+ CachedFileMetadataEntry::new(object_meta.clone(), Arc::clone(&metadata));+ cache.put(&object_meta.location, cached_entry);++ // Verify the cached value+ assert!(cache.contains_key(&object_meta.location));+ let result = cache.get(&object_meta.location).unwrap();+ let test_file_metadata = Arc::downcast::<TestFileMetadata>(result.file_metadata);+ assert!(test_file_metadata.is_ok());+ assert_eq!(test_file_metadata.unwrap().metadata, "retrieved_metadata");++ // Cache hit - check validation+ let result2 = cache.get(&object_meta.location).unwrap();+ assert!(result2.is_valid_for(&object_meta));++ // File size changed - closure should detect invalidity+ let object_meta2 = create_test_object_meta("test", 2048);+ let result3 = cache.get(&object_meta2.location).unwrap();+ // Cached entry should NOT be valid for new meta+ assert!(!result3.is_valid_for(&object_meta2));++ // Return new entry+ let new_entry =+ CachedFileMetadataEntry::new(object_meta2.clone(), Arc::clone(&metadata));+ cache.put(&object_meta2.location, new_entry);++ let result4 = cache.get(&object_meta2.location).unwrap();+ assert_eq!(result4.meta.size, 2048);++ // remove+ cache.remove(&object_meta.location);+ assert!(!cache.contains_key(&object_meta.location));++ // len and clear+ let object_meta3 = create_test_object_meta("test3", 100);+ cache.put(+ &object_meta.location,+ CachedFileMetadataEntry::new(object_meta.clone(), Arc::clone(&metadata)),+ );+ cache.put(+ &object_meta3.location,+ CachedFileMetadataEntry::new(object_meta3.clone(), Arc::clone(&metadata)),+ );+ assert_eq!(cache.len(), 2);+ cache.clear();+ assert_eq!(cache.len(), 0);+ }++ fn generate_test_metadata_with_size(+ path: &str,+ size: usize,+ ) -> (ObjectMeta, Arc<dyn FileMetadata>) {+ let object_meta = ObjectMeta {+ location: Path::from(path),+ last_modified: chrono::Utc::now(),+ size: size as u64,+ e_tag: None,+ version: None,+ };+ let metadata = "a".repeat(size);+ let metadata: Arc<dyn FileMetadata> = Arc::new(TestFileMetadata { metadata });++ (object_meta, metadata)+ }++ #[test]+ fn test_default_file_metadata_cache_with_limit() {+ // Create a cache with 1000 bytes capacity + 4 keys each key 2 bytes+ let cache = DefaultCache::new(1000 + 4 * 2);++ let (object_meta1, metadata1) = generate_test_metadata_with_size("01", 100);+ let (object_meta2, metadata2) = generate_test_metadata_with_size("02", 500);+ let (object_meta3, metadata3) = generate_test_metadata_with_size("03", 300);++ cache.put(+ &object_meta1.location,+ CachedFileMetadataEntry::new(object_meta1.clone(), metadata1),+ );+ cache.put(+ &object_meta2.location,+ CachedFileMetadataEntry::new(object_meta2.clone(), metadata2),+ );+ cache.put(+ &object_meta3.location,+ CachedFileMetadataEntry::new(object_meta3.clone(), metadata3),+ );++ // all entries will fit+ assert_eq!(cache.len(), 3);+ assert_eq!(cache.memory_used(), 906);+ assert!(cache.contains_key(&object_meta1.location));+ assert!(cache.contains_key(&object_meta2.location));+ assert!(cache.contains_key(&object_meta3.location));++ // add a new entry which will remove the least recently used ("1")+ let (object_meta4, metadata4) = generate_test_metadata_with_size("04", 200);+ cache.put(+ &object_meta4.location,+ CachedFileMetadataEntry::new(object_meta4.clone(), metadata4),+ );+ assert_eq!(cache.len(), 3);+ assert_eq!(cache.memory_used(), 1006);+ assert!(!cache.contains_key(&object_meta1.location));+ assert!(cache.contains_key(&object_meta4.location));++ // get entry "2", which will move it to the top of the queue, and add a new one which will+ // remove the new least recently used ("3")+ let _ = cache.get(&object_meta2.location);+ let (object_meta5, metadata5) = generate_test_metadata_with_size("05", 100);+ cache.put(+ &object_meta5.location,+ CachedFileMetadataEntry::new(object_meta5.clone(), metadata5),+ );+ assert_eq!(cache.len(), 3);+ assert_eq!(cache.memory_used(), 806);+ assert!(!cache.contains_key(&object_meta3.location));+ assert!(cache.contains_key(&object_meta5.location));++ // new entry which will not be able to fit in the 1000 bytes allocated+ let (object_meta6, metadata6) = generate_test_metadata_with_size("06", 1200);+ cache.put(+ &object_meta6.location,+ CachedFileMetadataEntry::new(object_meta6.clone(), metadata6),+ );+ assert_eq!(cache.len(), 3);+ assert_eq!(cache.memory_used(), 806);+ assert!(!cache.contains_key(&object_meta6.location));++ // new entry which is able to fit without removing any entry+ let (object_meta7, metadata7) = generate_test_metadata_with_size("07", 200);+ cache.put(+ &object_meta7.location,+ CachedFileMetadataEntry::new(object_meta7.clone(), metadata7),+ );+ assert_eq!(cache.len(), 4);+ assert_eq!(cache.memory_used(), 1008);+ assert!(cache.contains_key(&object_meta7.location));++ // new entry which will remove all other entries+ let (object_meta8, metadata8) = generate_test_metadata_with_size("08", 999);+ cache.put(+ &object_meta8.location,+ CachedFileMetadataEntry::new(object_meta8.clone(), metadata8),+ );+ assert_eq!(cache.len(), 1);+ assert_eq!(cache.memory_used(), 1001);+ assert!(cache.contains_key(&object_meta8.location));++ // when updating an entry, the previous ones are not unnecessarily removed+ let (object_meta9, metadata9) = generate_test_metadata_with_size("09", 300);+ let (object_meta10, metadata10) = generate_test_metadata_with_size("10", 200);+ let (object_meta11_v1, metadata11_v1) =+ generate_test_metadata_with_size("11", 400);+ cache.put(+ &object_meta9.location,+ CachedFileMetadataEntry::new(object_meta9.clone(), metadata9),+ );+ cache.put(+ &object_meta10.location,+ CachedFileMetadataEntry::new(object_meta10.clone(), metadata10),+ );+ cache.put(+ &object_meta11_v1.location,+ CachedFileMetadataEntry::new(object_meta11_v1.clone(), metadata11_v1),+ );+ assert_eq!(cache.memory_used(), 906);+ assert_eq!(cache.len(), 3);+ let (object_meta11_v2, metadata11_v2) =+ generate_test_metadata_with_size("11", 500);+ cache.put(+ &object_meta11_v2.location,+ CachedFileMetadataEntry::new(object_meta11_v2.clone(), metadata11_v2),+ );+ assert_eq!(cache.memory_used(), 1006);+ assert_eq!(cache.len(), 3);+ assert!(cache.contains_key(&object_meta9.location));+ assert!(cache.contains_key(&object_meta10.location));+ assert!(cache.contains_key(&object_meta11_v2.location));++ // when updating an entry that now exceeds the limit, the LRU ("09") needs to be removed+ let (object_meta11_v3, metadata11_v3) =+ generate_test_metadata_with_size("11", 510);+ cache.put(+ &object_meta11_v3.location,+ CachedFileMetadataEntry::new(object_meta11_v3.clone(), metadata11_v3),+ );+ assert_eq!(cache.memory_used(), 714);+ assert_eq!(cache.len(), 2);+ assert!(cache.contains_key(&object_meta10.location));+ assert!(cache.contains_key(&object_meta11_v3.location));++ // manually removing an entry that is not the LRU+ cache.remove(&object_meta11_v3.location);+ assert_eq!(cache.len(), 1);+ assert_eq!(cache.memory_used(), 202);+ assert!(cache.contains_key(&object_meta10.location));+ assert!(!cache.contains_key(&object_meta11_v3.location));++ // clear+ cache.clear();+ assert_eq!(cache.len(), 0);+ assert_eq!(cache.memory_used(), 0);++ // resizing the cache should clear the extra entries+ let (object_meta12, metadata12) = generate_test_metadata_with_size("12", 300);+ let (object_meta13, metadata13) = generate_test_metadata_with_size("13", 200);+ let (object_meta14, metadata14) = generate_test_metadata_with_size("14", 500);+ cache.put(+ &object_meta12.location,+ CachedFileMetadataEntry::new(object_meta12.clone(), metadata12),+ );+ cache.put(+ &object_meta13.location,+ CachedFileMetadataEntry::new(object_meta13.clone(), metadata13),+ );+ cache.put(+ &object_meta14.location,+ CachedFileMetadataEntry::new(object_meta14.clone(), metadata14),+ );+ assert_eq!(cache.len(), 3);+ assert_eq!(cache.memory_used(), 1006);+ cache.update_cache_limit(600);+ assert_eq!(cache.len(), 1);+ assert_eq!(cache.memory_used(), 502);+ assert!(!cache.contains_key(&object_meta12.location));+ assert!(!cache.contains_key(&object_meta13.location));+ assert!(cache.contains_key(&object_meta14.location));+ }++ #[test]+ fn test_default_file_metadata_cache_entries_info() {+ // Create a cache with 1000 bytes + 4 bytes for 4 keys each key 1 byte+ let cache = DefaultCache::new(1000 + 4);++ let (object_meta1, metadata1) = generate_test_metadata_with_size("1", 100);+ let (object_meta2, metadata2) = generate_test_metadata_with_size("2", 200);+ let (object_meta3, metadata3) = generate_test_metadata_with_size("3", 300);++ // initial entries, all will have hits = 0+ let entry_1 = CachedFileMetadataEntry::new(object_meta1.clone(), metadata1);+ let entry_2 = CachedFileMetadataEntry::new(object_meta2.clone(), metadata2);+ let entry_3 = CachedFileMetadataEntry::new(object_meta3.clone(), metadata3);++ // Build a cache which fits exactly these 3 entries++ cache.put(&object_meta1.location, entry_1.clone());+ cache.put(&object_meta2.location, entry_2.clone());+ cache.put(&object_meta3.location, entry_3.clone());+ let entries = cache.list_entries();++ assert_eq!(+ entries,+ HashMap::from([+ (+ Path::from("1"),+ CacheEntryInfo {+ value: entry_1.clone(),+ size_bytes: 100,+ hits: 0,+ expires: None,+ }+ ),+ (+ Path::from("2"),+ CacheEntryInfo {+ value: entry_2.clone(),+ size_bytes: 200,+ hits: 0,+ expires: None,+ }+ ),+ (+ Path::from("3"),+ CacheEntryInfo {+ value: entry_3.clone(),+ size_bytes: 300,+ hits: 0,+ expires: None,+ }+ )+ ])+ );++ // new hit on "1"+ let _ = cache.get(&object_meta1.location);+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ Path::from("1"),+ CacheEntryInfo {+ value: entry_1.clone(),+ size_bytes: 100,+ hits: 1,+ expires: None,+ }+ ),+ (+ Path::from("2"),+ CacheEntryInfo {+ value: entry_2.clone(),+ size_bytes: 200,+ hits: 0,+ expires: None,+ }+ ),+ (+ Path::from("3"),+ CacheEntryInfo {+ value: entry_3.clone(),+ size_bytes: 300,+ hits: 0,+ expires: None,+ }+ )+ ])+ );++ // new entry, will evict "2"+ let (object_meta4, metadata4) = generate_test_metadata_with_size("4", 600);+ let entry_4 = CachedFileMetadataEntry::new(object_meta4.clone(), metadata4);+ cache.put(&object_meta4.location, entry_4.clone());+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ Path::from("1"),+ CacheEntryInfo {+ value: entry_1.clone(),+ size_bytes: 100,+ hits: 1,+ expires: None,+ }+ ),+ (+ Path::from("3"),+ CacheEntryInfo {+ value: entry_3.clone(),+ size_bytes: 300,+ hits: 0,+ expires: None,+ }+ ),+ (+ Path::from("4"),+ CacheEntryInfo {+ value: entry_4.clone(),+ size_bytes: 600,+ hits: 0,+ expires: None,+ }+ )+ ])+ );++ // replace entry "1"+ let (object_meta1_new, metadata1_new) = generate_test_metadata_with_size("1", 50);+ let entry_1 =+ CachedFileMetadataEntry::new(object_meta1_new.clone(), metadata1_new);+ cache.put(&object_meta1_new.location, entry_1.clone());+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ Path::from("1"),+ CacheEntryInfo {+ value: entry_1.clone(),+ size_bytes: 50,+ hits: 0,+ expires: None,+ }+ ),+ (+ Path::from("3"),+ CacheEntryInfo {+ value: entry_3.clone(),+ size_bytes: 300,+ hits: 0,+ expires: None,+ }+ ),+ (+ Path::from("4"),+ CacheEntryInfo {+ value: entry_4.clone(),+ size_bytes: 600,+ hits: 0,+ expires: None,+ }+ )+ ])+ );++ // remove entry "4"+ cache.remove(&object_meta4.location);+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ Path::from("1"),+ CacheEntryInfo {+ value: entry_1.clone(),+ size_bytes: 50,+ hits: 0,+ expires: None,+ }+ ),+ (+ Path::from("3"),+ CacheEntryInfo {+ value: entry_3.clone(),+ size_bytes: 300,+ hits: 0,+ expires: None,+ }+ )+ ])+ );++ // clear+ cache.clear();+ assert_eq!(cache.list_entries(), HashMap::from([]));+ }++ fn create_test_meta(path: &str, size: u64) -> ObjectMeta {+ ObjectMeta {+ location: Path::from(path),+ last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")+ .unwrap()+ .into(),+ size,+ e_tag: None,+ version: None,+ }+ }++ #[test]+ fn test_statistics_cache() {+ let meta = create_test_meta("test", 1024);+ let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);++ let schema = Schema::new(vec![Field::new(+ "test_column",+ DataType::Timestamp(TimeUnit::Second, None),+ false,+ )]);++ let path = TableScopedPath {+ path: meta.location.clone(),+ table: None,+ };++ // Cache miss+ assert!(cache.get(&path).is_none());++ // Put a value+ let cached_value = CachedFileMetadata::new(+ meta.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&path, cached_value);++ // Cache hit+ let result = cache.get(&path);+ assert!(result.is_some());++ let cached = result.unwrap();+ assert!(cached.is_valid_for(&meta));++ // File size changed - validation should fail+ let meta2 = create_test_meta("test", 2048);++ let path_2 = TableScopedPath {+ path: meta2.location.clone(),+ table: None,+ };++ let cached = cache.get(&path_2).unwrap();+ assert!(!cached.is_valid_for(&meta2));++ // Update with new value+ let cached_value2 = CachedFileMetadata::new(+ meta2.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&path_2, cached_value2);++ // Test list_entries+ let entries = cache.list_entries();+ assert_eq!(entries.len(), 1);++ let path_3 = TableScopedPath {+ path: Path::from("test"),+ table: None,+ };++ let entry = entries.get(&path_3).unwrap();+ assert_eq!(entry.value.meta.size, 2048); // Should be updated value+ }++ #[derive(Clone, Debug, PartialEq, Eq, Hash)]+ struct MockExpr {}++ impl std::fmt::Display for MockExpr {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "MockExpr")+ }+ }++ impl PhysicalExpr for MockExpr {+ fn data_type(+ &self,+ _input_schema: &Schema,+ ) -> datafusion_common::Result<DataType> {+ Ok(DataType::Int32)+ }++ fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result<bool> {+ Ok(false)+ }++ fn evaluate(+ &self,+ _batch: &RecordBatch,+ ) -> datafusion_common::Result<ColumnarValue> {+ unimplemented!()+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> datafusion_common::Result<Arc<dyn PhysicalExpr>> {+ assert!(children.is_empty());+ Ok(self)+ }++ fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "MockExpr")+ }+ }++ fn ordering() -> LexOrdering {+ let expr = Arc::new(MockExpr {}) as Arc<dyn PhysicalExpr>;+ LexOrdering::new(vec![PhysicalSortExpr::new_default(expr)]).unwrap()+ }++ #[test]+ fn test_ordering_cache() {+ let meta = create_test_meta("test.parquet", 100);+ let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);++ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);++ // Cache statistics with no ordering+ let cached_value = CachedFileMetadata::new(+ meta.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None, // No ordering yet+ );++ let path = TableScopedPath {+ path: meta.location.clone(),+ table: None,+ };++ cache.put(&path, cached_value);++ let result = cache.get(&path).unwrap();+ assert!(result.ordering.is_none());++ // Update to add ordering+ let mut cached = cache.get(&path).unwrap();+ if cached.is_valid_for(&meta) && cached.ordering.is_none() {+ cached.ordering = Some(ordering());+ }+ cache.put(&path, cached);++ let result2 = cache.get(&path).unwrap();+ assert!(result2.ordering.is_some());++ // Verify list_entries shows has_ordering = true+ let entries = cache.list_entries();+ assert_eq!(entries.len(), 1);+ assert!(entries.get(&path).unwrap().value.ordering.is_some());+ }++ #[test]+ fn test_cache_invalidation_on_file_modification() {+ let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);+ let path = TableScopedPath {+ path: Path::from("test.parquet"),+ table: None,+ };+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);++ let meta_v1 = create_test_meta("test.parquet", 100);++ // Cache initial value+ let cached_value = CachedFileMetadata::new(+ meta_v1.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&path, cached_value);++ // File modified (size changed)+ let meta_v2 = create_test_meta("test.parquet", 200);++ let cached = cache.get(&path).unwrap();+ // Should not be valid for new meta+ assert!(!cached.is_valid_for(&meta_v2));++ // Compute new value and update+ let new_cached = CachedFileMetadata::new(+ meta_v2.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );+ cache.put(&path, new_cached);++ // Should have new metadata+ let result = cache.get(&path).unwrap();+ assert_eq!(result.meta.size, 200);+ }++ #[test]+ fn test_ordering_cache_invalidation_on_file_modification() {+ let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);+ let path = TableScopedPath {+ path: Path::from("test.parquet"),+ table: None,+ };+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);++ // Cache with original metadata and ordering+ let meta_v1 = ObjectMeta {+ location: path.path.clone(),+ last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")+ .unwrap()+ .into(),+ size: 100,+ e_tag: None,+ version: None,+ };+ let ordering_v1 = ordering();+ let cached_v1 = CachedFileMetadata::new(+ meta_v1.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ Some(ordering_v1),+ );+ cache.put(&path, cached_v1);++ // Verify cached ordering is valid+ let cached = cache.get(&path).unwrap();+ assert!(cached.is_valid_for(&meta_v1));+ assert!(cached.ordering.is_some());++ // File modified (size changed)+ let meta_v2 = ObjectMeta {+ location: path.path.clone(),+ last_modified: DateTime::parse_from_rfc3339("2022-09-28T10:00:00+02:00")+ .unwrap()+ .into(),+ size: 200, // Changed+ e_tag: None,+ version: None,+ };++ // Cache entry exists but should be invalid for new metadata+ let cached = cache.get(&path).unwrap();+ assert!(!cached.is_valid_for(&meta_v2));++ // Cache new version with different ordering+ let ordering_v2 = ordering(); // New ordering instance+ let cached_v2 = CachedFileMetadata::new(+ meta_v2.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ Some(ordering_v2),+ );+ cache.put(&path, cached_v2);++ // Old metadata should be invalid+ let cached = cache.get(&path).unwrap();+ assert!(!cached.is_valid_for(&meta_v1));++ // New metadata should be valid+ assert!(cached.is_valid_for(&meta_v2));+ assert!(cached.ordering.is_some());+ }++ #[test]+ fn test_list_entries() {+ let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);++ let meta1 = create_test_meta("test1.parquet", 100);++ let cached_value_1 = CachedFileMetadata::new(+ meta1.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ None,+ );++ let path_1 = TableScopedPath {+ path: meta1.location.clone(),+ table: None,+ };++ cache.put(&path_1, cached_value_1.clone());+ let meta2 = create_test_meta("test2.parquet", 200);+ let cached_value_2 = CachedFileMetadata::new(+ meta2.clone(),+ Arc::new(Statistics::new_unknown(&schema)),+ Some(ordering()),+ );++ let path_2 = TableScopedPath {+ path: meta2.location.clone(),+ table: None,+ };++ cache.put(&path_2, cached_value_2.clone());++ let entries = cache.list_entries();+ assert_eq!(+ entries,+ HashMap::from([+ (+ path_1,+ CacheEntryInfo {+ value: cached_value_1,+ hits: 0,+ size_bytes: 373,+ expires: None,+ }+ ),+ (+ path_2,+ CacheEntryInfo {+ value: cached_value_2,+ hits: 0,+ size_bytes: 373,+ expires: None,+ }+ ),+ ])+ );+ }++ #[test]+ fn test_cache_entry_added_when_entries_are_within_cache_limit() {+ let (meta_1, value_1) =+ create_cached_file_metadata_with_stats("test1.parquet", 10);+ let (meta_2, value_2) =+ create_cached_file_metadata_with_stats("test2.parquet", 10);+ let (meta_3, value_3) =+ create_cached_file_metadata_with_stats("test3.parquet", 10);++ let mut ctx = DFHeapSizeCtx::default();++ let limit_for_2_entries = meta_1.location.as_ref().heap_size(&mut ctx)+ + value_1.heap_size(&mut ctx)+ + meta_2.location.as_ref().heap_size(&mut ctx)+ + value_2.heap_size(&mut ctx);++ // create a cache with a limit which fits exactly 2 entries+ let cache = DefaultCache::new(limit_for_2_entries);+ let path_1 = TableScopedPath {+ path: meta_1.location.clone(),+ table: None,+ };++ let path_2 = TableScopedPath {+ path: meta_2.location.clone(),+ table: None,+ };++ cache.put(&path_1, value_1.clone());+ cache.put(&path_2, value_2.clone());++ assert_eq!(cache.len(), 2);+ assert_eq!(cache.memory_used(), limit_for_2_entries);++ let result_1 = cache.get(&path_1);+ let result_2 = cache.get(&path_2);+ assert_eq!(result_1.unwrap(), value_1);+ assert_eq!(result_2.unwrap(), value_2);++ let path_3 = TableScopedPath {+ path: meta_3.location.clone(),+ table: None,+ };++ // adding the third entry evicts the first entry+ cache.put(&path_3, value_3.clone());+ assert_eq!(cache.len(), 2);+ assert_eq!(cache.memory_used(), limit_for_2_entries);++ let result_1 = cache.get(&path_1);+ assert!(result_1.is_none());++ let result_2 = cache.get(&path_2);+ let result_3 = cache.get(&path_3);++ assert_eq!(result_2.unwrap(), value_2);+ assert_eq!(result_3.unwrap(), value_3);++ // add the third entry again, making sure memory usage remains the same+ cache.put(&path_3, value_3.clone());+ assert_eq!(cache.memory_used(), limit_for_2_entries);+ cache.put(&path_3, value_3.clone());+ assert_eq!(cache.memory_used(), limit_for_2_entries);++ let mut ctx = DFHeapSizeCtx::default();+ cache.remove(&path_2);+ assert_eq!(cache.len(), 1);+ assert_eq!(+ cache.memory_used(),+ meta_3.location.as_ref().heap_size(&mut ctx) + value_3.heap_size(&mut ctx)+ );++ cache.clear();+ assert_eq!(cache.len(), 0);+ assert_eq!(cache.memory_used(), 0);+ }++ #[test]+ fn test_cache_rejects_entry_which_is_too_large() {+ let (meta, value_too_large) =+ create_cached_file_metadata_with_stats("test1.parquet", 10);+ let mut ctx = DFHeapSizeCtx::default();+ let limit_less_than_the_entry = value_too_large.clone().heap_size(&mut ctx) - 1;++ // create a cache with a size less than the entry+ let cache = DefaultCache::new(limit_less_than_the_entry);++ let path_1 = TableScopedPath {+ path: meta.location.clone(),+ table: None,+ };++ cache.put(&path_1, value_too_large.clone());++ assert_eq!(cache.len(), 0);+ assert_eq!(cache.memory_used(), 0);++ // Test stale entry is removed when oversized entry is added+ let (_, value_fits) = create_cached_file_metadata_with_stats("test1.parquet", 7);+ cache.put(&path_1, value_fits.clone());++ assert_eq!(cache.len(), 1);+ assert_eq!(cache.memory_used(), 1514);++ // now add an entry which is over the limit and make sure the old stale entry is removed+ let stale_entry = cache.put(&path_1, value_too_large.clone());+ assert_eq!(stale_entry, Some(value_fits));++ assert_eq!(cache.len(), 0);+ assert_eq!(cache.memory_used(), 0);+ }++ fn create_cached_file_metadata_with_stats(+ file_name: &str,+ series_size: i32,+ ) -> (ObjectMeta, CachedFileMetadata) {+ let series: Vec<i32> = (0..=series_size).collect();+ let values = Int32Array::from(series);+ let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, series_size + 1]));+ let field = Arc::new(Field::new_list_field(DataType::Int32, false));+ let list_array = ListArray::new(field, offsets, Arc::new(values), None);++ let column_statistics = ColumnStatistics {+ null_count: Precision::Exact(1),+ max_value: Precision::Exact(ScalarValue::List(Arc::new(list_array.clone()))),+ min_value: Precision::Exact(ScalarValue::List(Arc::new(list_array.clone()))),+ sum_value: Precision::Exact(ScalarValue::List(Arc::new(list_array.clone()))),+ distinct_count: Precision::Exact(10),+ byte_size: Precision::Absent,+ };++ let stats = Statistics {+ num_rows: Precision::Exact(100),+ total_byte_size: Precision::Exact(100),+ column_statistics: vec![column_statistics.clone()],+ };+ let mut ctx = DFHeapSizeCtx::default();+ let object_meta = create_test_meta(file_name, stats.heap_size(&mut ctx) as u64);+ let value =+ CachedFileMetadata::new(object_meta.clone(), Arc::new(stats.clone()), None);+ (object_meta, value)+ }++ struct MockTimeProvider {+ base: Instant,+ offset: Mutex<Duration>,+ }++ impl MockTimeProvider {+ fn new() -> Self {+ Self {+ base: Instant::now(),+ offset: Mutex::new(Duration::ZERO),+ }+ }++ fn inc(&self, duration: Duration) {+ let mut offset = self.offset.lock().unwrap();+ *offset += duration;+ }+ }++ impl TimeProvider for MockTimeProvider {+ fn now(&self) -> Instant {+ self.base + *self.offset.lock().unwrap()+ }+ }++ /// Helper function to create a test ObjectMeta with a specific path and location string size+ fn create_object_meta(path: &str, location_size: usize) -> ObjectMeta {+ // Create a location string of the desired size by padding with zeros+ let location_str = if location_size > path.len() {+ format!("{}{}", path, "0".repeat(location_size - path.len()))+ } else {+ path.to_string()+ };++ ObjectMeta {+ location: Path::from(location_str),+ last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")+ .unwrap()+ .into(),+ size: 1024,+ e_tag: None,+ version: None,+ }+ }++ /// Helper function to create a TableScopedPath and a CachedFileList with at least meta_size bytes+ fn create_test_list_files_entry(+ path: &str,+ count: usize,+ meta_size: usize,+ table: Option<TableReference>,+ ) -> (TableScopedPath, CachedFileList) {+ let key = TableScopedPath {+ table,+ path: Path::from(path),+ };+ let metas: Vec<ObjectMeta> = (0..count)+ .map(|i| create_object_meta(&format!("file{i}"), meta_size))+ .collect();+ let value = CachedFileList::new(metas);+ (key, value)+ }++ #[test]+ fn test_basic_operations() {+ let cache = DefaultCache::new(DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT);+ let table_ref = Some(TableReference::from("table"));+ let path = Path::from("test_path");+ let key = TableScopedPath {+ table: table_ref.clone(),+ path,+ };++ // Initially cache is empty+ assert!(!cache.contains_key(&key));+ assert_eq!(cache.len(), 0);++ // Cache miss - get returns None+ assert!(cache.get(&key).is_none());++ // Put a value+ let meta = create_test_object_meta("file1", 50);+ cache.put(&key, CachedFileList::new(vec![meta]));++ // Entry should be cached+ assert!(cache.contains_key(&key));+ assert_eq!(cache.len(), 1);+ let result = cache.get(&key).unwrap();+ assert_eq!(result.files.len(), 1);++ // Remove the entry+ let removed = cache.remove(&key).unwrap();+ assert_eq!(removed.files.len(), 1);+ assert!(!cache.contains_key(&key));+ assert_eq!(cache.len(), 0);++ // Put multiple entries+ let (key1, value1) =+ create_test_list_files_entry("path1", 2, 50, table_ref.clone());+ let (key2, value2) = create_test_list_files_entry("path2", 3, 50, table_ref);+ cache.put(&key1, value1.clone());+ cache.put(&key2, value2.clone());+ assert_eq!(cache.len(), 2);++ // List cache entries+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ key1.clone(),+ CacheEntryInfo {+ value: value1.clone(),+ size_bytes: value1.size(),+ hits: 0,+ expires: None,+ }+ ),+ (+ key2.clone(),+ CacheEntryInfo {+ value: value2.clone(),+ size_bytes: value2.size(),+ hits: 0,+ expires: None,+ }+ )+ ])+ );++ // Clear all entries+ cache.clear();+ assert_eq!(cache.len(), 0);+ assert!(!cache.contains_key(&key1));+ assert!(!cache.contains_key(&key2));+ }++ #[test]+ fn test_lru_eviction_basic() {+ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, table_ref.clone());+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 100, table_ref.clone());+ let (key3, value3) =+ create_test_list_files_entry("path3", 1, 100, table_ref.clone());++ let entry_size = key1.size() + value1.size();++ // Set cache limit to exactly fit all 3 entries+ let cache = DefaultCache::new(entry_size * 3);++ // All three entries should fit+ cache.put(&key1, value1);+ cache.put(&key2, value2);+ cache.put(&key3, value3);+ assert_eq!(cache.len(), 3);+ assert!(cache.contains_key(&key1));+ assert!(cache.contains_key(&key2));+ assert!(cache.contains_key(&key3));++ // Adding a new entry should evict path1 (LRU)+ let (key4, value4) = create_test_list_files_entry("path4", 1, 100, table_ref);+ cache.put(&key4, value4);++ assert_eq!(cache.len(), 3);+ assert!(!cache.contains_key(&key1)); // Evicted+ assert!(cache.contains_key(&key2));+ assert!(cache.contains_key(&key3));+ assert!(cache.contains_key(&key4));+ }++ #[test]+ fn test_lru_ordering_after_access() {+ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, table_ref.clone());+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 100, table_ref.clone());+ let (key3, value3) =+ create_test_list_files_entry("path3", 1, 100, table_ref.clone());++ // Set cache limit to fit exactly three entries+ let cache = DefaultCache::new((key1.size() + value1.size()) * 3);++ cache.put(&key1, value1);+ cache.put(&key2, value2);+ cache.put(&key3, value3);+ assert_eq!(cache.len(), 3);++ // Access path1 to move it to front (MRU)+ // Order is now: path2 (LRU), path3, path1 (MRU)+ let _ = cache.get(&key1);++ // Adding a new entry should evict path2 (the LRU)+ let (key4, value4) = create_test_list_files_entry("path4", 1, 100, table_ref);+ cache.put(&key4, value4);++ assert_eq!(cache.len(), 3);+ assert!(cache.contains_key(&key1)); // Still present (recently accessed)+ assert!(!cache.contains_key(&key2)); // Evicted (was LRU)+ assert!(cache.contains_key(&key3));+ assert!(cache.contains_key(&key4));+ }++ #[test]+ fn test_reject_too_large() {+ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, table_ref.clone());+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 100, table_ref.clone());++ // Set cache limit to fit both entries+ let cache = DefaultCache::new((key1.size() + value1.size()) * 2);++ cache.put(&key1, value1);+ cache.put(&key2, value2);+ assert_eq!(cache.len(), 2);++ // Try to add an entry that's too large to fit in the cache+ // The entry is not stored (too large)+ let (key_large, value_large) =+ create_test_list_files_entry("large", 1, 1000, table_ref);+ cache.put(&key_large, value_large);++ // Large entry should not be added+ assert!(!cache.contains_key(&key_large));+ assert_eq!(cache.len(), 2);+ assert!(cache.contains_key(&key1));+ assert!(cache.contains_key(&key2));+ }++ #[test]+ fn test_multiple_evictions() {+ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, table_ref.clone());+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 100, table_ref.clone());+ let (key3, value3) =+ create_test_list_files_entry("path3", 1, 100, table_ref.clone());++ let entry_size = key1.size() + value1.size();++ // Set cache limit for exactly 3 entries+ let cache = DefaultCache::new(entry_size * 3);++ cache.put(&key1, value1);+ cache.put(&key2, value2);+ cache.put(&key3, value3);+ assert_eq!(cache.len(), 3);++ // Add a large entry that requires evicting 2 entries+ let (key_large, value_large) =+ create_test_list_files_entry("large", 1, 200, table_ref);+ cache.put(&key_large, value_large);++ // path1 and path2 should be evicted (both LRU), path3 and path_large remain+ assert_eq!(cache.len(), 2);+ assert!(!cache.contains_key(&key1)); // Evicted+ assert!(!cache.contains_key(&key2)); // Evicted+ assert!(cache.contains_key(&key3));+ assert!(cache.contains_key(&key_large));+ }++ #[test]+ fn test_cache_limit_resize() {+ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, table_ref.clone());+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 100, table_ref.clone());+ let (key3, value3) = create_test_list_files_entry("path3", 1, 100, table_ref);++ let entry_size = key1.size() + value1.size();++ let cache = DefaultCache::new(entry_size * 3);++ // Add three entries+ cache.put(&key1, value1);+ cache.put(&key2, value2);+ cache.put(&key3, value3);+ assert_eq!(cache.len(), 3);++ // Resize cache to only fit one entry+ cache.update_cache_limit(entry_size);++ // Should keep only the most recent entry (path3, the MRU)+ assert_eq!(cache.len(), 1);+ assert!(cache.contains_key(&key3));+ // Earlier entries (LRU) should be evicted+ assert!(!cache.contains_key(&key1));+ assert!(!cache.contains_key(&key2));+ }++ #[test]+ fn test_entry_update_with_size_change() {+ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, table_ref.clone());+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 100, table_ref.clone());+ let (key3, value3_v1) =+ create_test_list_files_entry("path3", 1, 100, table_ref.clone());++ let entry_size = key1.size() + value1.size();++ let cache = DefaultCache::new(entry_size * 3);++ // Add three entries+ cache.put(&key1, value1);+ cache.put(&key2, value2.clone());+ cache.put(&key3, value3_v1);+ assert_eq!(cache.len(), 3);++ // Update path3 with same size - should not cause eviction+ let (_, value3_v2) =+ create_test_list_files_entry("path3", 1, 100, table_ref.clone());+ cache.put(&key3, value3_v2);++ assert_eq!(cache.len(), 3);+ assert!(cache.contains_key(&key1));+ assert!(cache.contains_key(&key2));+ assert!(cache.contains_key(&key3));++ // Update path3 with larger size that requires evicting path1 (LRU)+ let (_, value3_v3) = create_test_list_files_entry("path3", 1, 200, table_ref);+ cache.put(&key3, value3_v3.clone());++ assert_eq!(cache.len(), 2);+ assert!(!cache.contains_key(&key1)); // Evicted (was LRU)+ assert!(cache.contains_key(&key2));+ assert!(cache.contains_key(&key3));++ // List cache entries+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ key2,+ CacheEntryInfo {+ value: value2.clone(),+ size_bytes: value2.size(),+ hits: 0,+ expires: None,+ }+ ),+ (+ key3,+ CacheEntryInfo {+ value: value3_v3.clone(),+ size_bytes: value3_v3.size(),+ hits: 0,+ expires: None,+ }+ )+ ])+ );+ }++ #[test]+ fn test_cache_with_ttl() {+ let ttl = Duration::from_millis(100);++ let mock_time = Arc::new(MockTimeProvider::new());+ let cache = DefaultCache::new_with_ttl(10000, Some(ttl))+ .with_time_provider(Arc::clone(&mock_time) as Arc<dyn TimeProvider>);++ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 2, 50, table_ref.clone());+ let (key2, value2) = create_test_list_files_entry("path2", 2, 50, table_ref);+ cache.put(&key1, value1.clone());+ cache.put(&key2, value2.clone());++ // Entries should be accessible immediately+ assert!(cache.get(&key1).is_some());+ assert!(cache.get(&key2).is_some());+ // List cache entries+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ key1.clone(),+ CacheEntryInfo {+ value: value1.clone(),+ size_bytes: value1.size(),+ hits: 1,+ expires: mock_time.now().checked_add(ttl),+ }+ ),+ (+ key2.clone(),+ CacheEntryInfo {+ value: value2.clone(),+ size_bytes: value2.size(),+ hits: 1,+ expires: mock_time.now().checked_add(ttl),+ }+ )+ ])+ );+ // Wait for TTL to expire+ mock_time.inc(Duration::from_millis(150));++ // Entries should now return None when observed through contains_key+ assert!(!cache.contains_key(&key1));+ assert_eq!(cache.len(), 1); // key1 was removed by contains_key()+ assert!(!cache.contains_key(&key2));+ assert_eq!(cache.len(), 0); // key2 was removed by contains_key()+ }++ #[test]+ fn test_cache_with_ttl_and_lru() {+ let ttl = Duration::from_millis(200);++ let mock_time = Arc::new(MockTimeProvider::new());+ let cache = DefaultCache::new_with_ttl(1100, Some(ttl))+ .with_time_provider(Arc::clone(&mock_time) as Arc<dyn TimeProvider>);++ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 400, table_ref.clone());+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 400, table_ref.clone());++ let (key3, value3) = create_test_list_files_entry("path3", 1, 400, table_ref);+ cache.put(&key1, value1);+ mock_time.inc(Duration::from_millis(50));+ cache.put(&key2, value2);+ mock_time.inc(Duration::from_millis(50));++ // path3 should evict path1 due to size limit+ cache.put(&key3, value3);+ assert!(!cache.contains_key(&key1)); // Evicted by LRU+ assert!(cache.contains_key(&key2));+ assert!(cache.contains_key(&key3));++ mock_time.inc(Duration::from_millis(151));++ assert!(!cache.contains_key(&key2)); // Expired+ assert!(cache.contains_key(&key3)); // Still valid+ }++ #[test]+ fn test_ttl_expiration_in_get() {+ let ttl = Duration::from_millis(100);+ let cache = DefaultCache::new_with_ttl(10000, Some(ttl));++ let table_ref = Some(TableReference::from("table"));+ let (key, value) = create_test_list_files_entry("path", 2, 50, table_ref);++ // Cache the entry+ cache.put(&key, value.clone());++ // Entry should be accessible immediately+ let result = cache.get(&key);+ assert!(result.is_some());+ assert_eq!(result.unwrap().files.len(), 2);++ // Wait for TTL to expire+ thread::sleep(Duration::from_millis(150));++ // Get should return None because entry expired+ let result2 = cache.get(&key);+ assert!(result2.is_none());+ }++ #[test]+ fn test_meta_heap_bytes_calculation() {+ // Test with minimal ObjectMeta (no e_tag, no version)+ let meta1 = ObjectMeta {+ location: Path::from("test"),+ last_modified: chrono::Utc::now(),+ size: 100,+ e_tag: None,+ version: None,+ };+ assert_eq!(meta_heap_bytes(&meta1), 4); // Just the location string "test"++ // Test with e_tag+ let meta2 = ObjectMeta {+ location: Path::from("test"),+ last_modified: chrono::Utc::now(),+ size: 100,+ e_tag: Some("etag123".to_string()),+ version: None,+ };+ assert_eq!(meta_heap_bytes(&meta2), 4 + 7); // location (4) + e_tag (7)++ // Test with version+ let meta3 = ObjectMeta {+ location: Path::from("test"),+ last_modified: chrono::Utc::now(),+ size: 100,+ e_tag: None,+ version: Some("v1.0".to_string()),+ };+ assert_eq!(meta_heap_bytes(&meta3), 4 + 4); // location (4) + version (4)++ // Test with both e_tag and version+ let meta4 = ObjectMeta {+ location: Path::from("test"),+ last_modified: chrono::Utc::now(),+ size: 100,+ e_tag: Some("tag".to_string()),+ version: Some("ver".to_string()),+ };+ assert_eq!(meta_heap_bytes(&meta4), 4 + 3 + 3); // location (4) + e_tag (3) + version (3)+ }++ #[test]+ fn test_memory_tracking() {+ let cache = DefaultCache::new(1000);++ // Verify cache starts with 0 memory used+ {+ assert_eq!(cache.memory_used(), 0);+ }++ // Add entry and verify memory tracking+ let table_ref = Some(TableReference::from("table"));+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, table_ref.clone());+ cache.put(&key1, value1.clone());+ let entry_size_1 = key1.size() + value1.size();+ {+ assert_eq!(cache.memory_used(), entry_size_1);+ }++ // Add another entry+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 200, table_ref.clone());+ cache.put(&key2, value2.clone());+ let entry_size_2 = key2.size() + value2.size();++ {+ assert_eq!(cache.memory_used(), entry_size_1 + entry_size_2);+ }++ // Remove first entry and verify memory decreases+ cache.remove(&key1);+ {+ assert_eq!(cache.memory_used(), entry_size_2);+ }++ // Clear and verify memory is 0+ cache.clear();+ {+ assert_eq!(cache.memory_used(), 0);+ }+ }++ // Prefix filtering tests using CachedFileList::filter_by_prefix++ /// Helper function to create ObjectMeta with a specific location path+ fn create_object_meta_with_path(location: &str) -> ObjectMeta {+ ObjectMeta {+ location: Path::from(location),+ last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")+ .unwrap()+ .into(),+ size: 1024,+ e_tag: None,+ version: None,+ }+ }++ #[test]+ fn test_prefix_filtering() {+ let cache = DefaultCache::new(100000);++ // Create files for a partitioned table+ let table_base = Path::from("my_table");+ let files = vec![+ create_object_meta_with_path("my_table/a=1/file1.parquet"),+ create_object_meta_with_path("my_table/a=1/file2.parquet"),+ create_object_meta_with_path("my_table/a=2/file3.parquet"),+ create_object_meta_with_path("my_table/a=2/file4.parquet"),+ ];++ // Cache the full table listing+ let table_ref = Some(TableReference::from("table"));+ let key = TableScopedPath {+ table: table_ref,+ path: table_base,+ };+ cache.put(&key, CachedFileList::new(files));++ let result = cache.get(&key).unwrap();++ // Filter for partition a=1+ let prefix_a1 = Some(Path::from("my_table/a=1"));+ let filtered = result.files_matching_prefix(&prefix_a1);+ assert_eq!(filtered.len(), 2);+ assert!(+ filtered+ .iter()+ .all(|m| m.location.as_ref().starts_with("my_table/a=1"))+ );++ // Filter for partition a=2+ let prefix_a2 = Some(Path::from("my_table/a=2"));+ let filtered_2 = result.files_matching_prefix(&prefix_a2);+ assert_eq!(filtered_2.len(), 2);+ assert!(+ filtered_2+ .iter()+ .all(|m| m.location.as_ref().starts_with("my_table/a=2"))+ );++ // No filter returns all+ let all = result.files_matching_prefix(&None);+ assert_eq!(all.len(), 4);+ }++ #[test]+ fn test_prefix_no_matching_files() {+ let cache = DefaultCache::new(100000);++ let table_base = Path::from("my_table");+ let files = vec![+ create_object_meta_with_path("my_table/a=1/file1.parquet"),+ create_object_meta_with_path("my_table/a=2/file2.parquet"),+ ];++ let table_ref = Some(TableReference::from("table"));+ let key = TableScopedPath {+ table: table_ref,+ path: table_base,+ };+ cache.put(&key, CachedFileList::new(files));+ let result = cache.get(&key).unwrap();++ // Query for partition a=3 which doesn't exist+ let prefix_a3 = Some(Path::from("my_table/a=3"));+ let filtered = result.files_matching_prefix(&prefix_a3);+ assert!(filtered.is_empty());+ }++ #[test]+ fn test_nested_partitions() {+ let cache = DefaultCache::new(100000);++ let table_base = Path::from("events");+ let files = vec![+ create_object_meta_with_path(+ "events/year=2024/month=01/day=01/file1.parquet",+ ),+ create_object_meta_with_path(+ "events/year=2024/month=01/day=02/file2.parquet",+ ),+ create_object_meta_with_path(+ "events/year=2024/month=02/day=01/file3.parquet",+ ),+ create_object_meta_with_path(+ "events/year=2025/month=01/day=01/file4.parquet",+ ),+ ];++ let table_ref = Some(TableReference::from("table"));+ let key = TableScopedPath {+ table: table_ref,+ path: table_base,+ };+ cache.put(&key, CachedFileList::new(files));+ let result = cache.get(&key).unwrap();++ // Filter for year=2024/month=01+ let prefix_month = Some(Path::from("events/year=2024/month=01"));+ let filtered = result.files_matching_prefix(&prefix_month);+ assert_eq!(filtered.len(), 2);++ // Filter for year=2024+ let prefix_year = Some(Path::from("events/year=2024"));+ let filtered_year = result.files_matching_prefix(&prefix_year);+ assert_eq!(filtered_year.len(), 3);+ }++ #[test]+ fn test_drop_table_entries() {+ let cache = DefaultCache::new(DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT);++ let table_ref1 = TableReference::from("table1");+ let table_ref2 = TableReference::from("table2");+ let (key1, value1) =+ create_test_list_files_entry("path1", 1, 100, Some(table_ref1.clone()));+ let (key2, value2) =+ create_test_list_files_entry("path2", 1, 100, Some(table_ref1.clone()));+ let (key3, value3) =+ create_test_list_files_entry("path3", 1, 100, Some(table_ref2.clone()));++ cache.put(&key1, value1);+ cache.put(&key2, value2);+ cache.put(&key3, value3);++ cache.drop_table_entries(&table_ref1).unwrap();++ assert!(!cache.contains_key(&key1));+ assert!(!cache.contains_key(&key2));+ assert!(cache.contains_key(&key3));+ }+}diff --git a/datafusion/execution/src/cache/file_metadata_cache.rs b/datafusion/execution/src/cache/file_metadata_cache.rsdeleted file mode 100644index e5c1e01e48..0000000000--- a/datafusion/execution/src/cache/file_metadata_cache.rs+++ /dev/null@@ -1,504 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--#[cfg(test)]-mod tests {- use std::sync::Arc;-- use crate::cache::cache_manager::{CachedFileMetadataEntry, FileMetadata};- use crate::cache::default_cache::DefaultCache;- use crate::cache::{Cache, CacheEntryInfo};- use datafusion_common::HashMap;- use object_store::ObjectMeta;- use object_store::path::Path;-- pub struct TestFileMetadata {- metadata: String,- }-- impl FileMetadata for TestFileMetadata {- fn as_any(&self) -> &dyn std::any::Any {- self- }-- fn memory_size(&self) -> usize {- self.metadata.len()- }-- fn extra_info(&self) -> HashMap<String, String> {- HashMap::from([("extra_info".to_owned(), "abc".to_owned())])- }- }-- impl PartialEq for CachedFileMetadataEntry {- fn eq(&self, other: &Self) -> bool {- self.meta == other.meta- }- }-- fn create_test_object_meta(path: &str, size: usize) -> ObjectMeta {- ObjectMeta {- location: Path::from(path),- last_modified: chrono::DateTime::parse_from_rfc3339(- "2025-07-29T12:12:12+00:00",- )- .unwrap()- .into(),- size: size as u64,- e_tag: None,- version: None,- }- }-- #[test]- fn test_default_file_metadata_cache() {- let object_meta = create_test_object_meta("test", 1024);-- let metadata: Arc<dyn FileMetadata> = Arc::new(TestFileMetadata {- metadata: "retrieved_metadata".to_owned(),- });-- let cache = DefaultCache::new(1024 * 1024);-- // Cache miss- assert!(cache.get(&object_meta.location).is_none());-- // Put a value- let cached_entry =- CachedFileMetadataEntry::new(object_meta.clone(), Arc::clone(&metadata));- cache.put(&object_meta.location, cached_entry);-- // Verify the cached value- assert!(cache.contains_key(&object_meta.location));- let result = cache.get(&object_meta.location).unwrap();- let test_file_metadata = Arc::downcast::<TestFileMetadata>(result.file_metadata);- assert!(test_file_metadata.is_ok());- assert_eq!(test_file_metadata.unwrap().metadata, "retrieved_metadata");-- // Cache hit - check validation- let result2 = cache.get(&object_meta.location).unwrap();- assert!(result2.is_valid_for(&object_meta));-- // File size changed - closure should detect invalidity- let object_meta2 = create_test_object_meta("test", 2048);- let result3 = cache.get(&object_meta2.location).unwrap();- // Cached entry should NOT be valid for new meta- assert!(!result3.is_valid_for(&object_meta2));-- // Return new entry- let new_entry =- CachedFileMetadataEntry::new(object_meta2.clone(), Arc::clone(&metadata));- cache.put(&object_meta2.location, new_entry);-- let result4 = cache.get(&object_meta2.location).unwrap();- assert_eq!(result4.meta.size, 2048);-- // remove- cache.remove(&object_meta.location);- assert!(!cache.contains_key(&object_meta.location));-- // len and clear- let object_meta3 = create_test_object_meta("test3", 100);- cache.put(- &object_meta.location,- CachedFileMetadataEntry::new(object_meta.clone(), Arc::clone(&metadata)),- );- cache.put(- &object_meta3.location,- CachedFileMetadataEntry::new(object_meta3.clone(), Arc::clone(&metadata)),- );- assert_eq!(cache.len(), 2);- cache.clear();- assert_eq!(cache.len(), 0);- }-- fn generate_test_metadata_with_size(- path: &str,- size: usize,- ) -> (ObjectMeta, Arc<dyn FileMetadata>) {- let object_meta = ObjectMeta {- location: Path::from(path),- last_modified: chrono::Utc::now(),- size: size as u64,- e_tag: None,- version: None,- };- let metadata = "a".repeat(size);- let metadata: Arc<dyn FileMetadata> = Arc::new(TestFileMetadata { metadata });-- (object_meta, metadata)- }-- #[test]- fn test_default_file_metadata_cache_with_limit() {- // Create a cache with 1000 bytes capacity + 4 keys each key 2 bytes- let cache = DefaultCache::new(1000 + 4 * 2);-- let (object_meta1, metadata1) = generate_test_metadata_with_size("01", 100);- let (object_meta2, metadata2) = generate_test_metadata_with_size("02", 500);- let (object_meta3, metadata3) = generate_test_metadata_with_size("03", 300);-- cache.put(- &object_meta1.location,- CachedFileMetadataEntry::new(object_meta1.clone(), metadata1),- );- cache.put(- &object_meta2.location,- CachedFileMetadataEntry::new(object_meta2.clone(), metadata2),- );- cache.put(- &object_meta3.location,- CachedFileMetadataEntry::new(object_meta3.clone(), metadata3),- );-- // all entries will fit- assert_eq!(cache.len(), 3);- assert_eq!(cache.memory_used(), 906);- assert!(cache.contains_key(&object_meta1.location));- assert!(cache.contains_key(&object_meta2.location));- assert!(cache.contains_key(&object_meta3.location));-- // add a new entry which will remove the least recently used ("1")- let (object_meta4, metadata4) = generate_test_metadata_with_size("04", 200);- cache.put(- &object_meta4.location,- CachedFileMetadataEntry::new(object_meta4.clone(), metadata4),- );- assert_eq!(cache.len(), 3);- assert_eq!(cache.memory_used(), 1006);- assert!(!cache.contains_key(&object_meta1.location));- assert!(cache.contains_key(&object_meta4.location));-- // get entry "2", which will move it to the top of the queue, and add a new one which will- // remove the new least recently used ("3")- let _ = cache.get(&object_meta2.location);- let (object_meta5, metadata5) = generate_test_metadata_with_size("05", 100);- cache.put(- &object_meta5.location,- CachedFileMetadataEntry::new(object_meta5.clone(), metadata5),- );- assert_eq!(cache.len(), 3);- assert_eq!(cache.memory_used(), 806);- assert!(!cache.contains_key(&object_meta3.location));- assert!(cache.contains_key(&object_meta5.location));-- // new entry which will not be able to fit in the 1000 bytes allocated- let (object_meta6, metadata6) = generate_test_metadata_with_size("06", 1200);- cache.put(- &object_meta6.location,- CachedFileMetadataEntry::new(object_meta6.clone(), metadata6),- );- assert_eq!(cache.len(), 3);- assert_eq!(cache.memory_used(), 806);- assert!(!cache.contains_key(&object_meta6.location));-- // new entry which is able to fit without removing any entry- let (object_meta7, metadata7) = generate_test_metadata_with_size("07", 200);- cache.put(- &object_meta7.location,- CachedFileMetadataEntry::new(object_meta7.clone(), metadata7),- );- assert_eq!(cache.len(), 4);- assert_eq!(cache.memory_used(), 1008);- assert!(cache.contains_key(&object_meta7.location));-- // new entry which will remove all other entries- let (object_meta8, metadata8) = generate_test_metadata_with_size("08", 999);- cache.put(- &object_meta8.location,- CachedFileMetadataEntry::new(object_meta8.clone(), metadata8),- );- assert_eq!(cache.len(), 1);- assert_eq!(cache.memory_used(), 1001);- assert!(cache.contains_key(&object_meta8.location));-- // when updating an entry, the previous ones are not unnecessarily removed- let (object_meta9, metadata9) = generate_test_metadata_with_size("09", 300);- let (object_meta10, metadata10) = generate_test_metadata_with_size("10", 200);- let (object_meta11_v1, metadata11_v1) =- generate_test_metadata_with_size("11", 400);- cache.put(- &object_meta9.location,- CachedFileMetadataEntry::new(object_meta9.clone(), metadata9),- );- cache.put(- &object_meta10.location,- CachedFileMetadataEntry::new(object_meta10.clone(), metadata10),- );- cache.put(- &object_meta11_v1.location,- CachedFileMetadataEntry::new(object_meta11_v1.clone(), metadata11_v1),- );- assert_eq!(cache.memory_used(), 906);- assert_eq!(cache.len(), 3);- let (object_meta11_v2, metadata11_v2) =- generate_test_metadata_with_size("11", 500);- cache.put(- &object_meta11_v2.location,- CachedFileMetadataEntry::new(object_meta11_v2.clone(), metadata11_v2),- );- assert_eq!(cache.memory_used(), 1006);- assert_eq!(cache.len(), 3);- assert!(cache.contains_key(&object_meta9.location));- assert!(cache.contains_key(&object_meta10.location));- assert!(cache.contains_key(&object_meta11_v2.location));-- // when updating an entry that now exceeds the limit, the LRU ("09") needs to be removed- let (object_meta11_v3, metadata11_v3) =- generate_test_metadata_with_size("11", 510);- cache.put(- &object_meta11_v3.location,- CachedFileMetadataEntry::new(object_meta11_v3.clone(), metadata11_v3),- );- assert_eq!(cache.memory_used(), 714);- assert_eq!(cache.len(), 2);- assert!(cache.contains_key(&object_meta10.location));- assert!(cache.contains_key(&object_meta11_v3.location));-- // manually removing an entry that is not the LRU- cache.remove(&object_meta11_v3.location);- assert_eq!(cache.len(), 1);- assert_eq!(cache.memory_used(), 202);- assert!(cache.contains_key(&object_meta10.location));- assert!(!cache.contains_key(&object_meta11_v3.location));-- // clear- cache.clear();- assert_eq!(cache.len(), 0);- assert_eq!(cache.memory_used(), 0);-- // resizing the cache should clear the extra entries- let (object_meta12, metadata12) = generate_test_metadata_with_size("12", 300);- let (object_meta13, metadata13) = generate_test_metadata_with_size("13", 200);- let (object_meta14, metadata14) = generate_test_metadata_with_size("14", 500);- cache.put(- &object_meta12.location,- CachedFileMetadataEntry::new(object_meta12.clone(), metadata12),- );- cache.put(- &object_meta13.location,- CachedFileMetadataEntry::new(object_meta13.clone(), metadata13),- );- cache.put(- &object_meta14.location,- CachedFileMetadataEntry::new(object_meta14.clone(), metadata14),- );- assert_eq!(cache.len(), 3);- assert_eq!(cache.memory_used(), 1006);- cache.update_cache_limit(600);- assert_eq!(cache.len(), 1);- assert_eq!(cache.memory_used(), 502);- assert!(!cache.contains_key(&object_meta12.location));- assert!(!cache.contains_key(&object_meta13.location));- assert!(cache.contains_key(&object_meta14.location));- }-- #[test]- fn test_default_file_metadata_cache_entries_info() {- // Create a cache with 1000 bytes + 4 bytes for 4 keys each key 1 byte- let cache = DefaultCache::new(1000 + 4);-- let (object_meta1, metadata1) = generate_test_metadata_with_size("1", 100);- let (object_meta2, metadata2) = generate_test_metadata_with_size("2", 200);- let (object_meta3, metadata3) = generate_test_metadata_with_size("3", 300);-- // initial entries, all will have hits = 0- let entry_1 = CachedFileMetadataEntry::new(object_meta1.clone(), metadata1);- let entry_2 = CachedFileMetadataEntry::new(object_meta2.clone(), metadata2);- let entry_3 = CachedFileMetadataEntry::new(object_meta3.clone(), metadata3);-- // Build a cache which fits exactly these 3 entries-- cache.put(&object_meta1.location, entry_1.clone());- cache.put(&object_meta2.location, entry_2.clone());- cache.put(&object_meta3.location, entry_3.clone());- let entries = cache.list_entries();-- assert_eq!(- entries,- HashMap::from([- (- Path::from("1"),- CacheEntryInfo {- value: entry_1.clone(),- size_bytes: 100,- hits: 0,- expires: None,- }- ),- (- Path::from("2"),- CacheEntryInfo {- value: entry_2.clone(),- size_bytes: 200,- hits: 0,- expires: None,- }- ),- (- Path::from("3"),- CacheEntryInfo {- value: entry_3.clone(),- size_bytes: 300,- hits: 0,- expires: None,- }- )- ])- );-- // new hit on "1"- let _ = cache.get(&object_meta1.location);- assert_eq!(- cache.list_entries(),- HashMap::from([- (- Path::from("1"),- CacheEntryInfo {- value: entry_1.clone(),- size_bytes: 100,- hits: 1,- expires: None,- }- ),- (- Path::from("2"),- CacheEntryInfo {- value: entry_2.clone(),- size_bytes: 200,- hits: 0,- expires: None,- }- ),- (- Path::from("3"),- CacheEntryInfo {- value: entry_3.clone(),- size_bytes: 300,- hits: 0,- expires: None,- }- )- ])- );-- // new entry, will evict "2"- let (object_meta4, metadata4) = generate_test_metadata_with_size("4", 600);- let entry_4 = CachedFileMetadataEntry::new(object_meta4.clone(), metadata4);- cache.put(&object_meta4.location, entry_4.clone());- assert_eq!(- cache.list_entries(),- HashMap::from([- (- Path::from("1"),- CacheEntryInfo {- value: entry_1.clone(),- size_bytes: 100,- hits: 1,- expires: None,- }- ),- (- Path::from("3"),- CacheEntryInfo {- value: entry_3.clone(),- size_bytes: 300,- hits: 0,- expires: None,- }- ),- (- Path::from("4"),- CacheEntryInfo {- value: entry_4.clone(),- size_bytes: 600,- hits: 0,- expires: None,- }- )- ])- );-- // replace entry "1"- let (object_meta1_new, metadata1_new) = generate_test_metadata_with_size("1", 50);- let entry_1 =- CachedFileMetadataEntry::new(object_meta1_new.clone(), metadata1_new);- cache.put(&object_meta1_new.location, entry_1.clone());- assert_eq!(- cache.list_entries(),- HashMap::from([- (- Path::from("1"),- CacheEntryInfo {- value: entry_1.clone(),- size_bytes: 50,- hits: 0,- expires: None,- }- ),- (- Path::from("3"),- CacheEntryInfo {- value: entry_3.clone(),- size_bytes: 300,- hits: 0,- expires: None,- }- ),- (- Path::from("4"),- CacheEntryInfo {- value: entry_4.clone(),- size_bytes: 600,- hits: 0,- expires: None,- }- )- ])- );-- // remove entry "4"- cache.remove(&object_meta4.location);- assert_eq!(- cache.list_entries(),- HashMap::from([- (- Path::from("1"),- CacheEntryInfo {- value: entry_1.clone(),- size_bytes: 50,- hits: 0,- expires: None,- }- ),- (- Path::from("3"),- CacheEntryInfo {- value: entry_3.clone(),- size_bytes: 300,- hits: 0,- expires: None,- }- )- ])- );-- // clear- cache.clear();- assert_eq!(cache.list_entries(), HashMap::from([]));- }-}diff --git a/datafusion/execution/src/cache/file_statistics_cache.rs b/datafusion/execution/src/cache/file_statistics_cache.rsdeleted file mode 100644index 5fb828d68d..0000000000--- a/datafusion/execution/src/cache/file_statistics_cache.rs+++ /dev/null@@ -1,512 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--#[cfg(test)]-mod tests {- use crate::cache::cache_manager::{- CachedFileMetadata, DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,- };- use crate::cache::default_cache::DefaultCache;- use crate::cache::{Cache, CacheEntryInfo, TableScopedPath};- use arrow::array::{Int32Array, ListArray, RecordBatch};- use arrow::buffer::{OffsetBuffer, ScalarBuffer};- use arrow::datatypes::{DataType, Field, Schema, TimeUnit};- use chrono::DateTime;- use datafusion_common::heap_size::{DFHeapSize, DFHeapSizeCtx};- use datafusion_common::stats::Precision;- use datafusion_common::{ColumnStatistics, HashMap, ScalarValue, Statistics};- use datafusion_expr::ColumnarValue;- use datafusion_physical_expr_common::physical_expr::PhysicalExpr;- use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};- use object_store::ObjectMeta;- use object_store::path::Path;- use std::sync::Arc;-- fn create_test_meta(path: &str, size: u64) -> ObjectMeta {- ObjectMeta {- location: Path::from(path),- last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")- .unwrap()- .into(),- size,- e_tag: None,- version: None,- }- }-- #[test]- fn test_statistics_cache() {- let meta = create_test_meta("test", 1024);- let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);-- let schema = Schema::new(vec![Field::new(- "test_column",- DataType::Timestamp(TimeUnit::Second, None),- false,- )]);-- let path = TableScopedPath {- path: meta.location.clone(),- table: None,- };-- // Cache miss- assert!(cache.get(&path).is_none());-- // Put a value- let cached_value = CachedFileMetadata::new(- meta.clone(),- Arc::new(Statistics::new_unknown(&schema)),- None,- );- cache.put(&path, cached_value);-- // Cache hit- let result = cache.get(&path);- assert!(result.is_some());-- let cached = result.unwrap();- assert!(cached.is_valid_for(&meta));-- // File size changed - validation should fail- let meta2 = create_test_meta("test", 2048);-- let path_2 = TableScopedPath {- path: meta2.location.clone(),- table: None,- };-- let cached = cache.get(&path_2).unwrap();- assert!(!cached.is_valid_for(&meta2));-- // Update with new value- let cached_value2 = CachedFileMetadata::new(- meta2.clone(),- Arc::new(Statistics::new_unknown(&schema)),- None,- );- cache.put(&path_2, cached_value2);-- // Test list_entries- let entries = cache.list_entries();- assert_eq!(entries.len(), 1);-- let path_3 = TableScopedPath {- path: Path::from("test"),- table: None,- };-- let entry = entries.get(&path_3).unwrap();- assert_eq!(entry.value.meta.size, 2048); // Should be updated value- }-- #[derive(Clone, Debug, PartialEq, Eq, Hash)]- struct MockExpr {}-- impl std::fmt::Display for MockExpr {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(f, "MockExpr")- }- }-- impl PhysicalExpr for MockExpr {- fn data_type(- &self,- _input_schema: &Schema,- ) -> datafusion_common::Result<DataType> {- Ok(DataType::Int32)- }-- fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result<bool> {- Ok(false)- }-- fn evaluate(- &self,- _batch: &RecordBatch,- ) -> datafusion_common::Result<ColumnarValue> {- unimplemented!()- }-- fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {- vec![]- }-- fn with_new_children(- self: Arc<Self>,- children: Vec<Arc<dyn PhysicalExpr>>,- ) -> datafusion_common::Result<Arc<dyn PhysicalExpr>> {- assert!(children.is_empty());- Ok(self)- }-- fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(f, "MockExpr")- }- }-- fn ordering() -> LexOrdering {- let expr = Arc::new(MockExpr {}) as Arc<dyn PhysicalExpr>;- LexOrdering::new(vec![PhysicalSortExpr::new_default(expr)]).unwrap()- }-- #[test]- fn test_ordering_cache() {- let meta = create_test_meta("test.parquet", 100);- let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);-- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);-- // Cache statistics with no ordering- let cached_value = CachedFileMetadata::new(- meta.clone(),- Arc::new(Statistics::new_unknown(&schema)),- None, // No ordering yet- );-- let path = TableScopedPath {- path: meta.location.clone(),- table: None,- };-- cache.put(&path, cached_value);-- let result = cache.get(&path).unwrap();- assert!(result.ordering.is_none());-- // Update to add ordering- let mut cached = cache.get(&path).unwrap();- if cached.is_valid_for(&meta) && cached.ordering.is_none() {- cached.ordering = Some(ordering());- }- cache.put(&path, cached);-- let result2 = cache.get(&path).unwrap();- assert!(result2.ordering.is_some());-- // Verify list_entries shows has_ordering = true- let entries = cache.list_entries();- assert_eq!(entries.len(), 1);- assert!(entries.get(&path).unwrap().value.ordering.is_some());- }-- #[test]- fn test_cache_invalidation_on_file_modification() {- let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);- let path = TableScopedPath {- path: Path::from("test.parquet"),- table: None,- };- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);-- let meta_v1 = create_test_meta("test.parquet", 100);-- // Cache initial value- let cached_value = CachedFileMetadata::new(- meta_v1.clone(),- Arc::new(Statistics::new_unknown(&schema)),- None,- );- cache.put(&path, cached_value);-- // File modified (size changed)- let meta_v2 = create_test_meta("test.parquet", 200);-- let cached = cache.get(&path).unwrap();- // Should not be valid for new meta- assert!(!cached.is_valid_for(&meta_v2));-- // Compute new value and update- let new_cached = CachedFileMetadata::new(- meta_v2.clone(),- Arc::new(Statistics::new_unknown(&schema)),- None,- );- cache.put(&path, new_cached);-- // Should have new metadata- let result = cache.get(&path).unwrap();- assert_eq!(result.meta.size, 200);- }-- #[test]- fn test_ordering_cache_invalidation_on_file_modification() {- let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);- let path = TableScopedPath {- path: Path::from("test.parquet"),- table: None,- };- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);-- // Cache with original metadata and ordering- let meta_v1 = ObjectMeta {- location: path.path.clone(),- last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")- .unwrap()- .into(),- size: 100,- e_tag: None,- version: None,- };- let ordering_v1 = ordering();- let cached_v1 = CachedFileMetadata::new(- meta_v1.clone(),- Arc::new(Statistics::new_unknown(&schema)),- Some(ordering_v1),- );- cache.put(&path, cached_v1);-- // Verify cached ordering is valid- let cached = cache.get(&path).unwrap();- assert!(cached.is_valid_for(&meta_v1));- assert!(cached.ordering.is_some());-- // File modified (size changed)- let meta_v2 = ObjectMeta {- location: path.path.clone(),- last_modified: DateTime::parse_from_rfc3339("2022-09-28T10:00:00+02:00")- .unwrap()- .into(),- size: 200, // Changed- e_tag: None,- version: None,- };-- // Cache entry exists but should be invalid for new metadata- let cached = cache.get(&path).unwrap();- assert!(!cached.is_valid_for(&meta_v2));-- // Cache new version with different ordering- let ordering_v2 = ordering(); // New ordering instance- let cached_v2 = CachedFileMetadata::new(- meta_v2.clone(),- Arc::new(Statistics::new_unknown(&schema)),- Some(ordering_v2),- );- cache.put(&path, cached_v2);-- // Old metadata should be invalid- let cached = cache.get(&path).unwrap();- assert!(!cached.is_valid_for(&meta_v1));-- // New metadata should be valid- assert!(cached.is_valid_for(&meta_v2));- assert!(cached.ordering.is_some());- }-- #[test]- fn test_list_entries() {- let cache = DefaultCache::new(DEFAULT_FILE_STATISTICS_MEMORY_LIMIT);- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);-- let meta1 = create_test_meta("test1.parquet", 100);-- let cached_value_1 = CachedFileMetadata::new(- meta1.clone(),- Arc::new(Statistics::new_unknown(&schema)),- None,- );-- let path_1 = TableScopedPath {- path: meta1.location.clone(),- table: None,- };-- cache.put(&path_1, cached_value_1.clone());- let meta2 = create_test_meta("test2.parquet", 200);- let cached_value_2 = CachedFileMetadata::new(- meta2.clone(),- Arc::new(Statistics::new_unknown(&schema)),- Some(ordering()),- );-- let path_2 = TableScopedPath {- path: meta2.location.clone(),- table: None,- };-- cache.put(&path_2, cached_value_2.clone());-- let entries = cache.list_entries();- assert_eq!(- entries,- HashMap::from([- (- path_1,- CacheEntryInfo {- value: cached_value_1,- hits: 0,- size_bytes: 373,- expires: None,- }- ),- (- path_2,- CacheEntryInfo {- value: cached_value_2,- hits: 0,- size_bytes: 373,- expires: None,- }- ),- ])- );- }-- #[test]- fn test_cache_entry_added_when_entries_are_within_cache_limit() {- let (meta_1, value_1) =- create_cached_file_metadata_with_stats("test1.parquet", 10);- let (meta_2, value_2) =- create_cached_file_metadata_with_stats("test2.parquet", 10);- let (meta_3, value_3) =- create_cached_file_metadata_with_stats("test3.parquet", 10);-- let mut ctx = DFHeapSizeCtx::default();-- let limit_for_2_entries = meta_1.location.as_ref().heap_size(&mut ctx)- + value_1.heap_size(&mut ctx)- + meta_2.location.as_ref().heap_size(&mut ctx)- + value_2.heap_size(&mut ctx);-- // create a cache with a limit which fits exactly 2 entries- let cache = DefaultCache::new(limit_for_2_entries);- let path_1 = TableScopedPath {- path: meta_1.location.clone(),- table: None,- };-- let path_2 = TableScopedPath {- path: meta_2.location.clone(),- table: None,- };-- cache.put(&path_1, value_1.clone());- cache.put(&path_2, value_2.clone());-- assert_eq!(cache.len(), 2);- assert_eq!(cache.memory_used(), limit_for_2_entries);-- let result_1 = cache.get(&path_1);- let result_2 = cache.get(&path_2);- assert_eq!(result_1.unwrap(), value_1);- assert_eq!(result_2.unwrap(), value_2);-- let path_3 = TableScopedPath {- path: meta_3.location.clone(),- table: None,- };-- // adding the third entry evicts the first entry- cache.put(&path_3, value_3.clone());- assert_eq!(cache.len(), 2);- assert_eq!(cache.memory_used(), limit_for_2_entries);-- let result_1 = cache.get(&path_1);- assert!(result_1.is_none());-- let result_2 = cache.get(&path_2);- let result_3 = cache.get(&path_3);-- assert_eq!(result_2.unwrap(), value_2);- assert_eq!(result_3.unwrap(), value_3);-- // add the third entry again, making sure memory usage remains the same- cache.put(&path_3, value_3.clone());- assert_eq!(cache.memory_used(), limit_for_2_entries);- cache.put(&path_3, value_3.clone());- assert_eq!(cache.memory_used(), limit_for_2_entries);-- let mut ctx = DFHeapSizeCtx::default();- cache.remove(&path_2);- assert_eq!(cache.len(), 1);- assert_eq!(- cache.memory_used(),- meta_3.location.as_ref().heap_size(&mut ctx) + value_3.heap_size(&mut ctx)- );-- cache.clear();- assert_eq!(cache.len(), 0);- assert_eq!(cache.memory_used(), 0);- }-- #[test]- fn test_cache_rejects_entry_which_is_too_large() {- let (meta, value_too_large) =- create_cached_file_metadata_with_stats("test1.parquet", 10);- let mut ctx = DFHeapSizeCtx::default();- let limit_less_than_the_entry = value_too_large.clone().heap_size(&mut ctx) - 1;-- // create a cache with a size less than the entry- let cache = DefaultCache::new(limit_less_than_the_entry);-- let path_1 = TableScopedPath {- path: meta.location.clone(),- table: None,- };-- cache.put(&path_1, value_too_large.clone());-- assert_eq!(cache.len(), 0);- assert_eq!(cache.memory_used(), 0);-- // Test stale entry is removed when oversized entry is added- let (_, value_fits) = create_cached_file_metadata_with_stats("test1.parquet", 7);- cache.put(&path_1, value_fits.clone());-- assert_eq!(cache.len(), 1);- assert_eq!(cache.memory_used(), 1514);-- // now add an entry which is over the limit and make sure the old stale entry is removed- let stale_entry = cache.put(&path_1, value_too_large.clone());- assert_eq!(stale_entry, Some(value_fits));-- assert_eq!(cache.len(), 0);- assert_eq!(cache.memory_used(), 0);- }-- fn create_cached_file_metadata_with_stats(- file_name: &str,- series_size: i32,- ) -> (ObjectMeta, CachedFileMetadata) {- let series: Vec<i32> = (0..=series_size).collect();- let values = Int32Array::from(series);- let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, series_size + 1]));- let field = Arc::new(Field::new_list_field(DataType::Int32, false));- let list_array = ListArray::new(field, offsets, Arc::new(values), None);-- let column_statistics = ColumnStatistics {- null_count: Precision::Exact(1),- max_value: Precision::Exact(ScalarValue::List(Arc::new(list_array.clone()))),- min_value: Precision::Exact(ScalarValue::List(Arc::new(list_array.clone()))),- sum_value: Precision::Exact(ScalarValue::List(Arc::new(list_array.clone()))),- distinct_count: Precision::Exact(10),- byte_size: Precision::Absent,- };-- let stats = Statistics {- num_rows: Precision::Exact(100),- total_byte_size: Precision::Exact(100),- column_statistics: vec![column_statistics.clone()],- };- let mut ctx = DFHeapSizeCtx::default();- let object_meta = create_test_meta(file_name, stats.heap_size(&mut ctx) as u64);- let value =- CachedFileMetadata::new(object_meta.clone(), Arc::new(stats.clone()), None);- (object_meta, value)- }-}diff --git a/datafusion/execution/src/cache/list_files_cache.rs b/datafusion/execution/src/cache/list_files_cache.rsdeleted file mode 100644index 968454fe45..0000000000--- a/datafusion/execution/src/cache/list_files_cache.rs+++ /dev/null@@ -1,736 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--#[cfg(test)]-mod tests {- use crate::cache::cache_manager::{- CachedFileList, DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, meta_heap_bytes,- };- use crate::cache::default_cache::{DefaultCache, TimeProvider};- use crate::cache::{Cache, CacheEntryInfo, CacheKey, CacheValue, TableScopedPath};- use chrono::DateTime;- use datafusion_common::HashMap;- use datafusion_common::TableReference;- use datafusion_common::instant::Instant;- use object_store::{ObjectMeta, path::Path};- use std::sync::{Arc, Mutex};- use std::thread;- use std::time::Duration;-- struct MockTimeProvider {- base: Instant,- offset: Mutex<Duration>,- }-- impl MockTimeProvider {- fn new() -> Self {- Self {- base: Instant::now(),- offset: Mutex::new(Duration::ZERO),- }- }-- fn inc(&self, duration: Duration) {- let mut offset = self.offset.lock().unwrap();- *offset += duration;- }- }-- impl TimeProvider for MockTimeProvider {- fn now(&self) -> Instant {- self.base + *self.offset.lock().unwrap()- }- }-- /// Helper function to create a test ObjectMeta with a specific path and location string size- fn create_test_object_meta(path: &str, location_size: usize) -> ObjectMeta {- // Create a location string of the desired size by padding with zeros- let location_str = if location_size > path.len() {- format!("{}{}", path, "0".repeat(location_size - path.len()))- } else {- path.to_string()- };-- ObjectMeta {- location: Path::from(location_str),- last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")- .unwrap()- .into(),- size: 1024,- e_tag: None,- version: None,- }- }-- /// Helper function to create a TableScopedPath and a CachedFileList with at least meta_size bytes- fn create_test_list_files_entry(- path: &str,- count: usize,- meta_size: usize,- table: Option<TableReference>,- ) -> (TableScopedPath, CachedFileList) {- let key = TableScopedPath {- table,- path: Path::from(path),- };- let metas: Vec<ObjectMeta> = (0..count)- .map(|i| create_test_object_meta(&format!("file{i}"), meta_size))- .collect();- let value = CachedFileList::new(metas);- (key, value)- }-- #[test]- fn test_basic_operations() {- let cache = DefaultCache::new(DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT);- let table_ref = Some(TableReference::from("table"));- let path = Path::from("test_path");- let key = TableScopedPath {- table: table_ref.clone(),- path,- };-- // Initially cache is empty- assert!(!cache.contains_key(&key));- assert_eq!(cache.len(), 0);-- // Cache miss - get returns None- assert!(cache.get(&key).is_none());-- // Put a value- let meta = create_test_object_meta("file1", 50);- cache.put(&key, CachedFileList::new(vec![meta]));-- // Entry should be cached- assert!(cache.contains_key(&key));- assert_eq!(cache.len(), 1);- let result = cache.get(&key).unwrap();- assert_eq!(result.files.len(), 1);-- // Remove the entry- let removed = cache.remove(&key).unwrap();- assert_eq!(removed.files.len(), 1);- assert!(!cache.contains_key(&key));- assert_eq!(cache.len(), 0);-- // Put multiple entries- let (key1, value1) =- create_test_list_files_entry("path1", 2, 50, table_ref.clone());- let (key2, value2) = create_test_list_files_entry("path2", 3, 50, table_ref);- cache.put(&key1, value1.clone());- cache.put(&key2, value2.clone());- assert_eq!(cache.len(), 2);-- // List cache entries- assert_eq!(- cache.list_entries(),- HashMap::from([- (- key1.clone(),- CacheEntryInfo {- value: value1.clone(),- size_bytes: value1.size(),- hits: 0,- expires: None,- }- ),- (- key2.clone(),- CacheEntryInfo {- value: value2.clone(),- size_bytes: value2.size(),- hits: 0,- expires: None,- }- )- ])- );-- // Clear all entries- cache.clear();- assert_eq!(cache.len(), 0);- assert!(!cache.contains_key(&key1));- assert!(!cache.contains_key(&key2));- }-- #[test]- fn test_lru_eviction_basic() {- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, table_ref.clone());- let (key2, value2) =- create_test_list_files_entry("path2", 1, 100, table_ref.clone());- let (key3, value3) =- create_test_list_files_entry("path3", 1, 100, table_ref.clone());-- let entry_size = key1.size() + value1.size();-- // Set cache limit to exactly fit all 3 entries- let cache = DefaultCache::new(entry_size * 3);-- // All three entries should fit- cache.put(&key1, value1);- cache.put(&key2, value2);- cache.put(&key3, value3);- assert_eq!(cache.len(), 3);- assert!(cache.contains_key(&key1));- assert!(cache.contains_key(&key2));- assert!(cache.contains_key(&key3));-- // Adding a new entry should evict path1 (LRU)- let (key4, value4) = create_test_list_files_entry("path4", 1, 100, table_ref);- cache.put(&key4, value4);-- assert_eq!(cache.len(), 3);- assert!(!cache.contains_key(&key1)); // Evicted- assert!(cache.contains_key(&key2));- assert!(cache.contains_key(&key3));- assert!(cache.contains_key(&key4));- }-- #[test]- fn test_lru_ordering_after_access() {- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, table_ref.clone());- let (key2, value2) =- create_test_list_files_entry("path2", 1, 100, table_ref.clone());- let (key3, value3) =- create_test_list_files_entry("path3", 1, 100, table_ref.clone());-- // Set cache limit to fit exactly three entries- let cache = DefaultCache::new((key1.size() + value1.size()) * 3);-- cache.put(&key1, value1);- cache.put(&key2, value2);- cache.put(&key3, value3);- assert_eq!(cache.len(), 3);-- // Access path1 to move it to front (MRU)- // Order is now: path2 (LRU), path3, path1 (MRU)- let _ = cache.get(&key1);-- // Adding a new entry should evict path2 (the LRU)- let (key4, value4) = create_test_list_files_entry("path4", 1, 100, table_ref);- cache.put(&key4, value4);-- assert_eq!(cache.len(), 3);- assert!(cache.contains_key(&key1)); // Still present (recently accessed)- assert!(!cache.contains_key(&key2)); // Evicted (was LRU)- assert!(cache.contains_key(&key3));- assert!(cache.contains_key(&key4));- }-- #[test]- fn test_reject_too_large() {- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, table_ref.clone());- let (key2, value2) =- create_test_list_files_entry("path2", 1, 100, table_ref.clone());-- // Set cache limit to fit both entries- let cache = DefaultCache::new((key1.size() + value1.size()) * 2);-- cache.put(&key1, value1);- cache.put(&key2, value2);- assert_eq!(cache.len(), 2);-- // Try to add an entry that's too large to fit in the cache- // The entry is not stored (too large)- let (key_large, value_large) =- create_test_list_files_entry("large", 1, 1000, table_ref);- cache.put(&key_large, value_large);-- // Large entry should not be added- assert!(!cache.contains_key(&key_large));- assert_eq!(cache.len(), 2);- assert!(cache.contains_key(&key1));- assert!(cache.contains_key(&key2));- }-- #[test]- fn test_multiple_evictions() {- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, table_ref.clone());- let (key2, value2) =- create_test_list_files_entry("path2", 1, 100, table_ref.clone());- let (key3, value3) =- create_test_list_files_entry("path3", 1, 100, table_ref.clone());-- let entry_size = key1.size() + value1.size();-- // Set cache limit for exactly 3 entries- let cache = DefaultCache::new(entry_size * 3);-- cache.put(&key1, value1);- cache.put(&key2, value2);- cache.put(&key3, value3);- assert_eq!(cache.len(), 3);-- // Add a large entry that requires evicting 2 entries- let (key_large, value_large) =- create_test_list_files_entry("large", 1, 200, table_ref);- cache.put(&key_large, value_large);-- // path1 and path2 should be evicted (both LRU), path3 and path_large remain- assert_eq!(cache.len(), 2);- assert!(!cache.contains_key(&key1)); // Evicted- assert!(!cache.contains_key(&key2)); // Evicted- assert!(cache.contains_key(&key3));- assert!(cache.contains_key(&key_large));- }-- #[test]- fn test_cache_limit_resize() {- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, table_ref.clone());- let (key2, value2) =- create_test_list_files_entry("path2", 1, 100, table_ref.clone());- let (key3, value3) = create_test_list_files_entry("path3", 1, 100, table_ref);-- let entry_size = key1.size() + value1.size();-- let cache = DefaultCache::new(entry_size * 3);-- // Add three entries- cache.put(&key1, value1);- cache.put(&key2, value2);- cache.put(&key3, value3);- assert_eq!(cache.len(), 3);-- // Resize cache to only fit one entry- cache.update_cache_limit(entry_size);-- // Should keep only the most recent entry (path3, the MRU)- assert_eq!(cache.len(), 1);- assert!(cache.contains_key(&key3));- // Earlier entries (LRU) should be evicted- assert!(!cache.contains_key(&key1));- assert!(!cache.contains_key(&key2));- }-- #[test]- fn test_entry_update_with_size_change() {- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, table_ref.clone());- let (key2, value2) =- create_test_list_files_entry("path2", 1, 100, table_ref.clone());- let (key3, value3_v1) =- create_test_list_files_entry("path3", 1, 100, table_ref.clone());-- let entry_size = key1.size() + value1.size();-- let cache = DefaultCache::new(entry_size * 3);-- // Add three entries- cache.put(&key1, value1);- cache.put(&key2, value2.clone());- cache.put(&key3, value3_v1);- assert_eq!(cache.len(), 3);-- // Update path3 with same size - should not cause eviction- let (_, value3_v2) =- create_test_list_files_entry("path3", 1, 100, table_ref.clone());- cache.put(&key3, value3_v2);-- assert_eq!(cache.len(), 3);- assert!(cache.contains_key(&key1));- assert!(cache.contains_key(&key2));- assert!(cache.contains_key(&key3));-- // Update path3 with larger size that requires evicting path1 (LRU)- let (_, value3_v3) = create_test_list_files_entry("path3", 1, 200, table_ref);- cache.put(&key3, value3_v3.clone());-- assert_eq!(cache.len(), 2);- assert!(!cache.contains_key(&key1)); // Evicted (was LRU)- assert!(cache.contains_key(&key2));- assert!(cache.contains_key(&key3));-- // List cache entries- assert_eq!(- cache.list_entries(),- HashMap::from([- (- key2,- CacheEntryInfo {- value: value2.clone(),- size_bytes: value2.size(),- hits: 0,- expires: None,- }- ),- (- key3,- CacheEntryInfo {- value: value3_v3.clone(),- size_bytes: value3_v3.size(),- hits: 0,- expires: None,- }- )- ])- );- }-- #[test]- fn test_cache_with_ttl() {- let ttl = Duration::from_millis(100);-- let mock_time = Arc::new(MockTimeProvider::new());- let cache = DefaultCache::new_with_ttl(10000, Some(ttl))- .with_time_provider(Arc::clone(&mock_time) as Arc<dyn TimeProvider>);-- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 2, 50, table_ref.clone());- let (key2, value2) = create_test_list_files_entry("path2", 2, 50, table_ref);- cache.put(&key1, value1.clone());- cache.put(&key2, value2.clone());-- // Entries should be accessible immediately- assert!(cache.get(&key1).is_some());- assert!(cache.get(&key2).is_some());- // List cache entries- assert_eq!(- cache.list_entries(),- HashMap::from([- (- key1.clone(),- CacheEntryInfo {- value: value1.clone(),- size_bytes: value1.size(),- hits: 1,- expires: mock_time.now().checked_add(ttl),- }- ),- (- key2.clone(),- CacheEntryInfo {- value: value2.clone(),- size_bytes: value2.size(),- hits: 1,- expires: mock_time.now().checked_add(ttl),- }- )- ])- );- // Wait for TTL to expire- mock_time.inc(Duration::from_millis(150));-- // Entries should now return None when observed through contains_key- assert!(!cache.contains_key(&key1));- assert_eq!(cache.len(), 1); // key1 was removed by contains_key()- assert!(!cache.contains_key(&key2));- assert_eq!(cache.len(), 0); // key2 was removed by contains_key()- }-- #[test]- fn test_cache_with_ttl_and_lru() {- let ttl = Duration::from_millis(200);-- let mock_time = Arc::new(MockTimeProvider::new());- let cache = DefaultCache::new_with_ttl(1100, Some(ttl))- .with_time_provider(Arc::clone(&mock_time) as Arc<dyn TimeProvider>);-- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 400, table_ref.clone());- let (key2, value2) =- create_test_list_files_entry("path2", 1, 400, table_ref.clone());-- let (key3, value3) = create_test_list_files_entry("path3", 1, 400, table_ref);- cache.put(&key1, value1);- mock_time.inc(Duration::from_millis(50));- cache.put(&key2, value2);- mock_time.inc(Duration::from_millis(50));-- // path3 should evict path1 due to size limit- cache.put(&key3, value3);- assert!(!cache.contains_key(&key1)); // Evicted by LRU- assert!(cache.contains_key(&key2));- assert!(cache.contains_key(&key3));-- mock_time.inc(Duration::from_millis(151));-- assert!(!cache.contains_key(&key2)); // Expired- assert!(cache.contains_key(&key3)); // Still valid- }-- #[test]- fn test_ttl_expiration_in_get() {- let ttl = Duration::from_millis(100);- let cache = DefaultCache::new_with_ttl(10000, Some(ttl));-- let table_ref = Some(TableReference::from("table"));- let (key, value) = create_test_list_files_entry("path", 2, 50, table_ref);-- // Cache the entry- cache.put(&key, value.clone());-- // Entry should be accessible immediately- let result = cache.get(&key);- assert!(result.is_some());- assert_eq!(result.unwrap().files.len(), 2);-- // Wait for TTL to expire- thread::sleep(Duration::from_millis(150));-- // Get should return None because entry expired- let result2 = cache.get(&key);- assert!(result2.is_none());- }-- #[test]- fn test_meta_heap_bytes_calculation() {- // Test with minimal ObjectMeta (no e_tag, no version)- let meta1 = ObjectMeta {- location: Path::from("test"),- last_modified: chrono::Utc::now(),- size: 100,- e_tag: None,- version: None,- };- assert_eq!(meta_heap_bytes(&meta1), 4); // Just the location string "test"-- // Test with e_tag- let meta2 = ObjectMeta {- location: Path::from("test"),- last_modified: chrono::Utc::now(),- size: 100,- e_tag: Some("etag123".to_string()),- version: None,- };- assert_eq!(meta_heap_bytes(&meta2), 4 + 7); // location (4) + e_tag (7)-- // Test with version- let meta3 = ObjectMeta {- location: Path::from("test"),- last_modified: chrono::Utc::now(),- size: 100,- e_tag: None,- version: Some("v1.0".to_string()),- };- assert_eq!(meta_heap_bytes(&meta3), 4 + 4); // location (4) + version (4)-- // Test with both e_tag and version- let meta4 = ObjectMeta {- location: Path::from("test"),- last_modified: chrono::Utc::now(),- size: 100,- e_tag: Some("tag".to_string()),- version: Some("ver".to_string()),- };- assert_eq!(meta_heap_bytes(&meta4), 4 + 3 + 3); // location (4) + e_tag (3) + version (3)- }-- #[test]- fn test_memory_tracking() {- let cache = DefaultCache::new(1000);-- // Verify cache starts with 0 memory used- {- assert_eq!(cache.memory_used(), 0);- }-- // Add entry and verify memory tracking- let table_ref = Some(TableReference::from("table"));- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, table_ref.clone());- cache.put(&key1, value1.clone());- let entry_size_1 = key1.size() + value1.size();- {- assert_eq!(cache.memory_used(), entry_size_1);- }-- // Add another entry- let (key2, value2) =- create_test_list_files_entry("path2", 1, 200, table_ref.clone());- cache.put(&key2, value2.clone());- let entry_size_2 = key2.size() + value2.size();-- {- assert_eq!(cache.memory_used(), entry_size_1 + entry_size_2);- }-- // Remove first entry and verify memory decreases- cache.remove(&key1);- {- assert_eq!(cache.memory_used(), entry_size_2);- }-- // Clear and verify memory is 0- cache.clear();- {- assert_eq!(cache.memory_used(), 0);- }- }-- // Prefix filtering tests using CachedFileList::filter_by_prefix-- /// Helper function to create ObjectMeta with a specific location path- fn create_object_meta_with_path(location: &str) -> ObjectMeta {- ObjectMeta {- location: Path::from(location),- last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00")- .unwrap()- .into(),- size: 1024,- e_tag: None,- version: None,- }- }-- #[test]- fn test_prefix_filtering() {- let cache = DefaultCache::new(100000);-- // Create files for a partitioned table- let table_base = Path::from("my_table");- let files = vec![- create_object_meta_with_path("my_table/a=1/file1.parquet"),- create_object_meta_with_path("my_table/a=1/file2.parquet"),- create_object_meta_with_path("my_table/a=2/file3.parquet"),- create_object_meta_with_path("my_table/a=2/file4.parquet"),- ];-- // Cache the full table listing- let table_ref = Some(TableReference::from("table"));- let key = TableScopedPath {- table: table_ref,- path: table_base,- };- cache.put(&key, CachedFileList::new(files));-- let result = cache.get(&key).unwrap();-- // Filter for partition a=1- let prefix_a1 = Some(Path::from("my_table/a=1"));- let filtered = result.files_matching_prefix(&prefix_a1);- assert_eq!(filtered.len(), 2);- assert!(- filtered- .iter()- .all(|m| m.location.as_ref().starts_with("my_table/a=1"))- );-- // Filter for partition a=2- let prefix_a2 = Some(Path::from("my_table/a=2"));- let filtered_2 = result.files_matching_prefix(&prefix_a2);- assert_eq!(filtered_2.len(), 2);- assert!(- filtered_2- .iter()- .all(|m| m.location.as_ref().starts_with("my_table/a=2"))- );-- // No filter returns all- let all = result.files_matching_prefix(&None);- assert_eq!(all.len(), 4);- }-- #[test]- fn test_prefix_no_matching_files() {- let cache = DefaultCache::new(100000);-- let table_base = Path::from("my_table");- let files = vec![- create_object_meta_with_path("my_table/a=1/file1.parquet"),- create_object_meta_with_path("my_table/a=2/file2.parquet"),- ];-- let table_ref = Some(TableReference::from("table"));- let key = TableScopedPath {- table: table_ref,- path: table_base,- };- cache.put(&key, CachedFileList::new(files));- let result = cache.get(&key).unwrap();-- // Query for partition a=3 which doesn't exist- let prefix_a3 = Some(Path::from("my_table/a=3"));- let filtered = result.files_matching_prefix(&prefix_a3);- assert!(filtered.is_empty());- }-- #[test]- fn test_nested_partitions() {- let cache = DefaultCache::new(100000);-- let table_base = Path::from("events");- let files = vec![- create_object_meta_with_path(- "events/year=2024/month=01/day=01/file1.parquet",- ),- create_object_meta_with_path(- "events/year=2024/month=01/day=02/file2.parquet",- ),- create_object_meta_with_path(- "events/year=2024/month=02/day=01/file3.parquet",- ),- create_object_meta_with_path(- "events/year=2025/month=01/day=01/file4.parquet",- ),- ];-- let table_ref = Some(TableReference::from("table"));- let key = TableScopedPath {- table: table_ref,- path: table_base,- };- cache.put(&key, CachedFileList::new(files));- let result = cache.get(&key).unwrap();-- // Filter for year=2024/month=01- let prefix_month = Some(Path::from("events/year=2024/month=01"));- let filtered = result.files_matching_prefix(&prefix_month);- assert_eq!(filtered.len(), 2);-- // Filter for year=2024- let prefix_year = Some(Path::from("events/year=2024"));- let filtered_year = result.files_matching_prefix(&prefix_year);- assert_eq!(filtered_year.len(), 3);- }-- #[test]- fn test_drop_table_entries() {- let cache = DefaultCache::new(DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT);-- let table_ref1 = TableReference::from("table1");- let table_ref2 = TableReference::from("table2");- let (key1, value1) =- create_test_list_files_entry("path1", 1, 100, Some(table_ref1.clone()));- let (key2, value2) =- create_test_list_files_entry("path2", 1, 100, Some(table_ref1.clone()));- let (key3, value3) =- create_test_list_files_entry("path3", 1, 100, Some(table_ref2.clone()));-- cache.put(&key1, value1);- cache.put(&key2, value2);- cache.put(&key3, value3);-- cache.drop_table_entries(&table_ref1).unwrap();-- assert!(!cache.contains_key(&key1));- assert!(!cache.contains_key(&key2));- assert!(cache.contains_key(&key3));- }-}diff --git a/datafusion/execution/src/cache/mod.rs b/datafusion/execution/src/cache/mod.rsindex 07a85142ba..49c2969587 100644--- a/datafusion/execution/src/cache/mod.rs+++ b/datafusion/execution/src/cache/mod.rs@@ -16,12 +16,9 @@// under the License.pub mod cache_manager;-mod file_statistics_cache;pub mod lru_queue;pub mod default_cache;-mod file_metadata_cache;-mod list_files_cache;use datafusion_common::heap_size::{DFHeapSize, DFHeapSizeCtx};use datafusion_common::instant::Instant;