Apache DataFusion SQL Query Engine
fix: preserve ListFilesCache TTL when not set in config (#19401)
Shashidhar B M887aa9f8c641
diff --git a/datafusion/execution/src/cache/cache_manager.rs b/datafusion/execution/src/cache/cache_manager.rsindex 89b48e9629..1ff35f2707 100644--- a/datafusion/execution/src/cache/cache_manager.rs+++ b/datafusion/execution/src/cache/cache_manager.rs@@ -196,7 +196,10 @@ impl CacheManager {.inspect(|c| {// the cache memory limit or ttl might have changed, ensure they are updatedc.update_cache_limit(config.list_files_cache_limit);- c.update_cache_ttl(config.list_files_cache_ttl);+ // Only update TTL if explicitly set in config, otherwise preserve the cache's existing TTL+ if let Some(ttl) = config.list_files_cache_ttl {+ c.update_cache_ttl(Some(ttl));+ }}).map(Arc::clone);@@ -350,3 +353,77 @@ impl CacheManagerConfig {self}}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::cache::DefaultListFilesCache;++ /// Test to verify that TTL is preserved when not explicitly set in config.+ /// This fixes issue #19396 where TTL was being unset from DefaultListFilesCache+ /// when CacheManagerConfig::list_files_cache_ttl was not set explicitly.+ #[test]+ fn test_ttl_preserved_when_not_set_in_config() {+ use std::time::Duration;++ // Create a cache with TTL = 1 second+ let list_file_cache =+ DefaultListFilesCache::new(1024, Some(Duration::from_secs(1)));++ // Verify the cache has TTL set initially+ assert_eq!(+ list_file_cache.cache_ttl(),+ Some(Duration::from_secs(1)),+ "Cache should have TTL = 1 second initially"+ );++ // Put cache in config WITHOUT setting list_files_cache_ttl+ let config = CacheManagerConfig::default()+ .with_list_files_cache(Some(Arc::new(list_file_cache)));++ // Create CacheManager from config+ let cache_manager = CacheManager::try_new(&config).unwrap();++ // Verify TTL is preserved (not unset)+ let cache_ttl = cache_manager.get_list_files_cache().unwrap().cache_ttl();++ assert!(+ cache_ttl.is_some(),+ "TTL should be preserved when not set in config. Expected Some(Duration::from_secs(1)), got {cache_ttl:?}"+ );++ // Verify it's the correct TTL value+ assert_eq!(+ cache_ttl,+ Some(Duration::from_secs(1)),+ "TTL should be exactly 1 second"+ );+ }++ /// Test to verify that TTL can still be overridden when explicitly set in config.+ #[test]+ fn test_ttl_overridden_when_set_in_config() {+ use std::time::Duration;++ // Create a cache with TTL = 1 second+ let list_file_cache =+ DefaultListFilesCache::new(1024, Some(Duration::from_secs(1)));++ // Put cache in config WITH a different TTL set+ let config = CacheManagerConfig::default()+ .with_list_files_cache(Some(Arc::new(list_file_cache)))+ .with_list_files_cache_ttl(Some(Duration::from_secs(60)));++ // Create CacheManager from config+ let cache_manager = CacheManager::try_new(&config).unwrap();++ // Verify TTL is overridden to the config value+ let cache_ttl = cache_manager.get_list_files_cache().unwrap().cache_ttl();++ assert_eq!(+ cache_ttl,+ Some(Duration::from_secs(60)),+ "TTL should be overridden to 60 seconds when set in config"+ );+ }+}