Apache DataFusion SQL Query Engine
Improvements to `list_files_cache` table function (#19703)
Andrew Lamb0808f3a8d264
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rsindex 6a97c5355f..e50339d296 100644--- a/datafusion-cli/src/functions.rs+++ b/datafusion-cli/src/functions.rs@@ -703,10 +703,13 @@ impl TableFunctionImpl for StatisticsCacheFunc {}}-// Implementation of the `list_files_cache` table function in datafusion-cli.+/// Implementation of the `list_files_cache` table function in datafusion-cli.+///+/// This function returns the cached results of running a LIST command on a+/// particular object store path for a table. The object metadata is returned as+/// a List of Structs, with one Struct for each object. DataFusion uses these+/// cached results to plan queries against external tables.///-/// This function returns the cached results of running a LIST command on a particular object store path for a table. The object metadata is returned as a List of Structs, with one Struct for each object.-/// DataFusion uses these cached results to plan queries against external tables./// # Schema/// ```sql/// > describe select * from list_files_cache();@@ -788,7 +791,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {Field::new("metadata", DataType::Struct(nested_fields.clone()), true);let schema = Arc::new(Schema::new(vec![- Field::new("table", DataType::Utf8, false),+ Field::new("table", DataType::Utf8, true),Field::new("path", DataType::Utf8, false),Field::new("metadata_size_bytes", DataType::UInt64, false),// expires field in ListFilesEntry has type Instant when set, from which we cannot get "the number of seconds", hence using Duration instead of Timestamp as data type.@@ -821,7 +824,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {let mut current_offset: i32 = 0;for (path, entry) in list_files_cache.list_entries() {- table_arr.push(path.table.map_or("NULL".to_string(), |t| t.to_string()));+ table_arr.push(path.table.map(|t| t.to_string()));path_arr.push(path.path.to_string());metadata_size_bytes_arr.push(entry.size_bytes as u64);// calculates time left before entry expiresdiff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rsindex 86af691fd7..98f61a8528 100644--- a/datafusion/core/src/datasource/listing_table_factory.rs+++ b/datafusion/core/src/datasource/listing_table_factory.rs@@ -161,9 +161,7 @@ impl TableProviderFactory for ListingTableFactory {}None => format!("*.{}", cmd.file_type.to_lowercase()),};- table_path = table_path- .with_glob(glob.as_ref())?- .with_table_ref(cmd.name.clone());+ table_path = table_path.with_glob(glob.as_ref())?;}let schema = options.infer_schema(session_state, &table_path).await?;let df_schema = Arc::clone(&schema).to_dfschema()?;diff --git a/datafusion/datasource/src/url.rs b/datafusion/datasource/src/url.rsindex 678bd280fc..0c274806c0 100644--- a/datafusion/datasource/src/url.rs+++ b/datafusion/datasource/src/url.rs@@ -43,7 +43,7 @@ pub struct ListingTableUrl {prefix: Path,/// An optional glob expression used to filter filesglob: Option<Pattern>,-+ /// Optional table reference for the table this url belongs totable_ref: Option<TableReference>,}@@ -341,17 +341,19 @@ impl ListingTableUrl {}/// Returns a copy of current [`ListingTableUrl`] with a specified `glob`- pub fn with_glob(self, glob: &str) -> Result<Self> {- let glob =- Pattern::new(glob).map_err(|e| DataFusionError::External(Box::new(e)))?;- Self::try_new(self.url, Some(glob))+ pub fn with_glob(mut self, glob: &str) -> Result<Self> {+ self.glob =+ Some(Pattern::new(glob).map_err(|e| DataFusionError::External(Box::new(e)))?);+ Ok(self)}+ /// Set the table reference for this [`ListingTableUrl`]pub fn with_table_ref(mut self, table_ref: TableReference) -> Self {self.table_ref = Some(table_ref);self}+ /// Return the table reference for this [`ListingTableUrl`]pub fn get_table_ref(&self) -> &Option<TableReference> {&self.table_ref}diff --git a/datafusion/execution/src/cache/cache_manager.rs b/datafusion/execution/src/cache/cache_manager.rsindex 4cc5586440..bd34c441bd 100644--- a/datafusion/execution/src/cache/cache_manager.rs+++ b/datafusion/execution/src/cache/cache_manager.rs@@ -196,6 +196,7 @@ pub trait ListFilesCache: CacheAccessor<TableScopedPath, CachedFileList> {/// Retrieves the information about the entries currently cached.fn list_entries(&self) -> HashMap<TableScopedPath, ListFilesEntry>;+ /// Drop all entries for the given table reference.fn drop_table_entries(&self, table_ref: &Option<TableReference>) -> Result<()>;}diff --git a/datafusion/execution/src/cache/list_files_cache.rs b/datafusion/execution/src/cache/list_files_cache.rsindex c86a03574e..b1b8e6b500 100644--- a/datafusion/execution/src/cache/list_files_cache.rs+++ b/datafusion/execution/src/cache/list_files_cache.rs@@ -139,6 +139,11 @@ pub const DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT: usize = 1024 * 1024; // 1MiB/// The default cache TTL for the [`DefaultListFilesCache`]pub const DEFAULT_LIST_FILES_CACHE_TTL: Option<Duration> = None; // Infinite+/// Key for [`DefaultListFilesCache`]+///+/// Each entry is scoped to its use within a specific table so that the cache+/// can differentiate between identical paths in different tables, and+/// table-level cache invalidation.#[derive(PartialEq, Eq, Hash, Clone, Debug)]pub struct TableScopedPath {pub table: Option<TableReference>,