Apache DataFusion SQL Query Engine
feat: add list_files_cache table function for `datafusion-cli` (#19388)
jizezhang1037f0aa205e
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rsindex a45d57e8e9..aa83fec111 100644--- a/datafusion-cli/src/functions.rs+++ b/datafusion-cli/src/functions.rs@@ -17,13 +17,18 @@//! Functions that are query-able and searchable via the `\h` command+use datafusion_common::instant::Instant;use std::fmt;use std::fs::File;use std::str::FromStr;use std::sync::Arc;-use arrow::array::{Int64Array, StringArray, TimestampMillisecondArray, UInt64Array};-use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit};+use arrow::array::{+ DurationMillisecondArray, GenericListArray, Int64Array, StringArray, StructArray,+ TimestampMillisecondArray, UInt64Array,+};+use arrow::buffer::{Buffer, OffsetBuffer, ScalarBuffer};+use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef, TimeUnit};use arrow::record_batch::RecordBatch;use arrow::util::pretty::pretty_format_batches;use datafusion::catalog::{Session, TableFunctionImpl};@@ -697,3 +702,158 @@ impl TableFunctionImpl for StatisticsCacheFunc {Ok(Arc::new(statistics_cache))}}++#[derive(Debug)]+struct ListFilesCacheTable {+ schema: SchemaRef,+ batch: RecordBatch,+}++#[async_trait]+impl TableProvider for ListFilesCacheTable {+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }++ fn schema(&self) -> arrow::datatypes::SchemaRef {+ self.schema.clone()+ }++ fn table_type(&self) -> datafusion::logical_expr::TableType {+ datafusion::logical_expr::TableType::Base+ }++ async fn scan(+ &self,+ _state: &dyn Session,+ projection: Option<&Vec<usize>>,+ _filters: &[Expr],+ _limit: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(MemorySourceConfig::try_new_exec(+ &[vec![self.batch.clone()]],+ TableProvider::schema(self),+ projection.cloned(),+ )?)+ }+}++#[derive(Debug)]+pub struct ListFilesCacheFunc {+ cache_manager: Arc<CacheManager>,+}++impl ListFilesCacheFunc {+ pub fn new(cache_manager: Arc<CacheManager>) -> Self {+ Self { cache_manager }+ }+}++impl TableFunctionImpl for ListFilesCacheFunc {+ fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ if !exprs.is_empty() {+ return plan_err!("list_files_cache should have no arguments");+ }++ let nested_fields = Fields::from(vec![+ Field::new("file_path", DataType::Utf8, false),+ Field::new(+ "file_modified",+ DataType::Timestamp(TimeUnit::Millisecond, None),+ false,+ ),+ Field::new("file_size_bytes", DataType::UInt64, false),+ Field::new("e_tag", DataType::Utf8, true),+ Field::new("version", DataType::Utf8, true),+ ]);++ let metadata_field =+ Field::new("metadata", DataType::Struct(nested_fields.clone()), true);++ let schema = Arc::new(Schema::new(vec![+ Field::new("path", DataType::Utf8, false),+ Field::new("metadata_size_bytes", DataType::UInt64, false),+ // expires field in ListFilesEntry has type Instant when set, from which we cannot get "the number of seconds", hence using Duration instead of Timestamp as data type.+ Field::new(+ "expires_in",+ DataType::Duration(TimeUnit::Millisecond),+ true,+ ),+ Field::new(+ "metadata_list",+ DataType::List(Arc::new(metadata_field.clone())),+ true,+ ),+ ]));++ let mut path_arr = vec![];+ let mut metadata_size_bytes_arr = vec![];+ let mut expires_arr = vec![];++ let mut file_path_arr = vec![];+ let mut file_modified_arr = vec![];+ let mut file_size_bytes_arr = vec![];+ let mut etag_arr = vec![];+ let mut version_arr = vec![];+ let mut offsets: Vec<i32> = vec![0];++ if let Some(list_files_cache) = self.cache_manager.get_list_files_cache() {+ let now = Instant::now();+ let mut current_offset: i32 = 0;++ for (path, entry) in list_files_cache.list_entries() {+ path_arr.push(path.to_string());+ metadata_size_bytes_arr.push(entry.size_bytes as u64);+ // calculates time left before entry expires+ expires_arr.push(+ entry+ .expires+ .map(|t| t.duration_since(now).as_millis() as i64),+ );++ for meta in entry.metas.iter() {+ file_path_arr.push(meta.location.to_string());+ file_modified_arr.push(meta.last_modified.timestamp_millis());+ file_size_bytes_arr.push(meta.size);+ etag_arr.push(meta.e_tag.clone());+ version_arr.push(meta.version.clone());+ }+ current_offset += entry.metas.len() as i32;+ offsets.push(current_offset);+ }+ }++ let struct_arr = StructArray::new(+ nested_fields,+ vec![+ Arc::new(StringArray::from(file_path_arr)),+ Arc::new(TimestampMillisecondArray::from(file_modified_arr)),+ Arc::new(UInt64Array::from(file_size_bytes_arr)),+ Arc::new(StringArray::from(etag_arr)),+ Arc::new(StringArray::from(version_arr)),+ ],+ None,+ );++ let offsets_buffer: OffsetBuffer<i32> =+ OffsetBuffer::new(ScalarBuffer::from(Buffer::from_vec(offsets)));++ let batch = RecordBatch::try_new(+ schema.clone(),+ vec![+ Arc::new(StringArray::from(path_arr)),+ Arc::new(UInt64Array::from(metadata_size_bytes_arr)),+ Arc::new(DurationMillisecondArray::from(expires_arr)),+ Arc::new(GenericListArray::new(+ Arc::new(metadata_field),+ offsets_buffer,+ Arc::new(struct_arr),+ None,+ )),+ ],+ )?;++ let list_files_cache = ListFilesCacheTable { schema, batch };+ Ok(Arc::new(list_files_cache))+ }+}diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rsindex 8f69ae4779..46d88152fa 100644--- a/datafusion-cli/src/main.rs+++ b/datafusion-cli/src/main.rs@@ -32,7 +32,7 @@ use datafusion::logical_expr::ExplainFormat;use datafusion::prelude::SessionContext;use datafusion_cli::catalog::DynamicObjectStoreCatalog;use datafusion_cli::functions::{- MetadataCacheFunc, ParquetMetadataFunc, StatisticsCacheFunc,+ ListFilesCacheFunc, MetadataCacheFunc, ParquetMetadataFunc, StatisticsCacheFunc,};use datafusion_cli::object_storage::instrumented::{InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry,@@ -253,6 +253,13 @@ async fn main_inner() -> Result<()> {)),);+ ctx.register_udtf(+ "list_files_cache",+ Arc::new(ListFilesCacheFunc::new(+ ctx.task_ctx().runtime_env().cache_manager.clone(),+ )),+ );+let mut print_options = PrintOptions {format: args.format,quiet: args.quiet,@@ -431,15 +438,20 @@ pub fn extract_disk_limit(size: &str) -> Result<usize, String> {#[cfg(test)]mod tests {+ use std::time::Duration;+use super::*;use datafusion::{common::test_util::batches_to_string,execution::cache::{- cache_manager::CacheManagerConfig, cache_unit::DefaultFileStatisticsCache,+ DefaultListFilesCache, cache_manager::CacheManagerConfig,+ cache_unit::DefaultFileStatisticsCache,},- prelude::ParquetReadOptions,+ prelude::{ParquetReadOptions, col, lit, split_part},};use insta::assert_snapshot;+ use object_store::memory::InMemory;+ use url::Url;fn assert_conversion(input: &str, expected: Result<usize, String>) {let result = extract_memory_pool_size(input);@@ -741,4 +753,130 @@ mod tests {Ok(())}++ #[tokio::test]+ async fn test_list_files_cache() -> Result<(), DataFusionError> {+ let list_files_cache = Arc::new(DefaultListFilesCache::new(+ 1024,+ Some(Duration::from_secs(1)),+ ));++ let rt = RuntimeEnvBuilder::new()+ .with_cache_manager(+ CacheManagerConfig::default()+ .with_list_files_cache(Some(list_files_cache)),+ )+ .build_arc()+ .unwrap();++ let ctx = SessionContext::new_with_config_rt(SessionConfig::default(), rt);++ ctx.register_object_store(+ &Url::parse("mem://test_table").unwrap(),+ Arc::new(InMemory::new()),+ );++ ctx.register_udtf(+ "list_files_cache",+ Arc::new(ListFilesCacheFunc::new(+ ctx.task_ctx().runtime_env().cache_manager.clone(),+ )),+ );++ ctx.sql(+ "CREATE EXTERNAL TABLE src_table+ STORED AS PARQUET+ LOCATION '../parquet-testing/data/alltypes_plain.parquet'",+ )+ .await?+ .collect()+ .await?;++ ctx.sql("COPY (SELECT * FROM src_table) TO 'mem://test_table/0.parquet' STORED AS PARQUET").await?.collect().await?;++ ctx.sql("COPY (SELECT * FROM src_table) TO 'mem://test_table/1.parquet' STORED AS PARQUET").await?.collect().await?;++ ctx.sql(+ "CREATE EXTERNAL TABLE test_table+ STORED AS PARQUET+ LOCATION 'mem://test_table/'+ ",+ )+ .await?+ .collect()+ .await?;++ let sql = "SELECT metadata_size_bytes, expires_in, metadata_list FROM list_files_cache()";+ let df = ctx+ .sql(sql)+ .await?+ .unnest_columns(&["metadata_list"])?+ .with_column_renamed("metadata_list", "metadata")?+ .unnest_columns(&["metadata"])?;++ assert_eq!(+ 2,+ df.clone()+ .filter(col("expires_in").is_not_null())?+ .count()+ .await?+ );++ let df = df+ .with_column_renamed(r#""metadata.file_size_bytes""#, "file_size_bytes")?+ .with_column_renamed(r#""metadata.e_tag""#, "etag")?+ .with_column(+ "filename",+ split_part(col(r#""metadata.file_path""#), lit("/"), lit(-1)),+ )?+ .select_columns(&[+ "metadata_size_bytes",+ "filename",+ "file_size_bytes",+ "etag",+ ])?+ .sort(vec![col("filename").sort(true, false)])?;+ let rbs = df.collect().await?;+ assert_snapshot!(batches_to_string(&rbs),@r"+ +---------------------+-----------+-----------------+------++ | metadata_size_bytes | filename | file_size_bytes | etag |+ +---------------------+-----------+-----------------+------++ | 212 | 0.parquet | 3645 | 0 |+ | 212 | 1.parquet | 3645 | 1 |+ +---------------------+-----------+-----------------+------++ ");++ Ok(())+ }++ #[tokio::test]+ async fn test_list_files_cache_not_set() -> Result<(), DataFusionError> {+ let rt = RuntimeEnvBuilder::new()+ .with_cache_manager(CacheManagerConfig::default().with_list_files_cache(None))+ .build_arc()+ .unwrap();++ let ctx = SessionContext::new_with_config_rt(SessionConfig::default(), rt);++ ctx.register_udtf(+ "list_files_cache",+ Arc::new(ListFilesCacheFunc::new(+ ctx.task_ctx().runtime_env().cache_manager.clone(),+ )),+ );++ let rbs = ctx+ .sql("SELECT * FROM list_files_cache()")+ .await?+ .collect()+ .await?;+ assert_snapshot!(batches_to_string(&rbs),@r"+ +------+---------------------+------------+---------------++ | path | metadata_size_bytes | expires_in | metadata_list |+ +------+---------------------+------------+---------------++ +------+---------------------+------------+---------------++ ");++ Ok(())+ }}diff --git a/datafusion/execution/src/cache/cache_manager.rs b/datafusion/execution/src/cache/cache_manager.rsindex c76a68c651..31a2323524 100644--- a/datafusion/execution/src/cache/cache_manager.rs+++ b/datafusion/execution/src/cache/cache_manager.rs@@ -16,6 +16,7 @@// under the License.use crate::cache::cache_unit::DefaultFilesMetadataCache;+use crate::cache::list_files_cache::ListFilesEntry;use crate::cache::{CacheAccessor, DefaultListFilesCache};use datafusion_common::stats::Precision;use datafusion_common::{Result, Statistics};@@ -93,6 +94,9 @@ pub trait ListFilesCache:/// Updates the cache with a new TTL (time-to-live).fn update_cache_ttl(&self, ttl: Option<Duration>);++ /// Retrieves the information about the entries currently cached.+ fn list_entries(&self) -> HashMap<Path, ListFilesEntry>;}/// Generic file-embedded metadata used with [`FileMetadataCache`].diff --git a/datafusion/execution/src/cache/list_files_cache.rs b/datafusion/execution/src/cache/list_files_cache.rsindex 661bc47b54..c4a92c4947 100644--- a/datafusion/execution/src/cache/list_files_cache.rs+++ b/datafusion/execution/src/cache/list_files_cache.rs@@ -17,6 +17,7 @@use std::mem::size_of;use std::{+ collections::HashMap,sync::{Arc, Mutex},time::Duration,};@@ -103,10 +104,11 @@ impl DefaultListFilesCache {}}-struct ListFilesEntry {- metas: Arc<Vec<ObjectMeta>>,- size_bytes: usize,- expires: Option<Instant>,+#[derive(Clone, PartialEq, Debug)]+pub struct ListFilesEntry {+ pub metas: Arc<Vec<ObjectMeta>>,+ pub size_bytes: usize,+ pub expires: Option<Instant>,}impl ListFilesEntry {@@ -347,6 +349,15 @@ impl ListFilesCache for DefaultListFilesCache {state.ttl = ttl;state.evict_entries();}++ fn list_entries(&self) -> HashMap<Path, ListFilesEntry> {+ let state = self.state.lock().unwrap();+ let mut entries = HashMap::<Path, ListFilesEntry>::new();+ for (path, entry) in state.lru_queue.list_entries() {+ entries.insert(path.clone(), entry.clone());+ }+ entries+ }}impl CacheAccessor<Path, Arc<Vec<ObjectMeta>>> for DefaultListFilesCache {@@ -431,7 +442,6 @@ impl CacheAccessor<Path, Arc<Vec<ObjectMeta>>> for DefaultListFilesCache {mod tests {use super::*;use chrono::DateTime;- use std::thread;struct MockTimeProvider {base: Instant,@@ -525,12 +535,35 @@ mod tests {assert_eq!(cache.len(), 0);// Put multiple entries- let (path1, value1, _) = create_test_list_files_entry("path1", 2, 50);- let (path2, value2, _) = create_test_list_files_entry("path2", 3, 50);- cache.put(&path1, value1);- cache.put(&path2, value2);+ let (path1, value1, size1) = create_test_list_files_entry("path1", 2, 50);+ let (path2, value2, size2) = create_test_list_files_entry("path2", 3, 50);+ cache.put(&path1, Arc::clone(&value1));+ cache.put(&path2, Arc::clone(&value2));assert_eq!(cache.len(), 2);+ // List cache entries+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ path1.clone(),+ ListFilesEntry {+ metas: value1,+ size_bytes: size1,+ expires: None,+ }+ ),+ (+ path2.clone(),+ ListFilesEntry {+ metas: value2,+ size_bytes: size2,+ expires: None,+ }+ )+ ])+ );+// Clear all entriescache.clear();assert_eq!(cache.len(), 0);@@ -673,14 +706,14 @@ mod tests {#[test]fn test_entry_update_with_size_change() {let (path1, value1, size) = create_test_list_files_entry("path1", 1, 100);- let (path2, value2, _) = create_test_list_files_entry("path2", 1, 100);+ let (path2, value2, size2) = create_test_list_files_entry("path2", 1, 100);let (path3, value3_v1, _) = create_test_list_files_entry("path3", 1, 100);let cache = DefaultListFilesCache::new(size * 3, None);// Add three entriescache.put(&path1, value1);- cache.put(&path2, value2);+ cache.put(&path2, Arc::clone(&value2));cache.put(&path3, value3_v1);assert_eq!(cache.len(), 3);@@ -694,35 +727,77 @@ mod tests {assert!(cache.contains_key(&path3));// Update path3 with larger size that requires evicting path1 (LRU)- let (_, value3_v3, _) = create_test_list_files_entry("path3", 1, 200);- cache.put(&path3, value3_v3);+ let (_, value3_v3, size3_v3) = create_test_list_files_entry("path3", 1, 200);+ cache.put(&path3, Arc::clone(&value3_v3));assert_eq!(cache.len(), 2);- assert!(!cache.contains_key(&path1)); // Evicted (was LRU)- assert!(cache.contains_key(&path2));- assert!(cache.contains_key(&path3));+ assert!(!cache.contains_key(&path1));++ // List cache entries+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ path2,+ ListFilesEntry {+ metas: value2,+ size_bytes: size2,+ expires: None,+ }+ ),+ (+ path3,+ ListFilesEntry {+ metas: value3_v3,+ size_bytes: size3_v3,+ expires: None,+ }+ )+ ])+ );}#[test]fn test_cache_with_ttl() {let ttl = Duration::from_millis(100);- let cache = DefaultListFilesCache::new(10000, Some(ttl));- let (path1, value1, _) = create_test_list_files_entry("path1", 2, 50);- let (path2, value2, _) = create_test_list_files_entry("path2", 2, 50);+ let mock_time = Arc::new(MockTimeProvider::new());+ let cache = DefaultListFilesCache::new(10000, Some(ttl))+ .with_time_provider(Arc::clone(&mock_time) as Arc<dyn TimeProvider>);- cache.put(&path1, value1);- cache.put(&path2, value2);+ let (path1, value1, size1) = create_test_list_files_entry("path1", 2, 50);+ let (path2, value2, size2) = create_test_list_files_entry("path2", 2, 50);++ cache.put(&path1, Arc::clone(&value1));+ cache.put(&path2, Arc::clone(&value2));// Entries should be accessible immediatelyassert!(cache.get(&path1).is_some());assert!(cache.get(&path2).is_some());- assert!(cache.contains_key(&path1));- assert!(cache.contains_key(&path2));- assert_eq!(cache.len(), 2);-+ // List cache entries+ assert_eq!(+ cache.list_entries(),+ HashMap::from([+ (+ path1.clone(),+ ListFilesEntry {+ metas: value1,+ size_bytes: size1,+ expires: mock_time.now().checked_add(ttl),+ }+ ),+ (+ path2.clone(),+ ListFilesEntry {+ metas: value2,+ size_bytes: size2,+ expires: mock_time.now().checked_add(ttl),+ }+ )+ ])+ );// Wait for TTL to expire- thread::sleep(Duration::from_millis(150));+ mock_time.inc(Duration::from_millis(150));// Entries should now return None and be removed when observed through get or contains_keyassert!(cache.get(&path1).is_none());diff --git a/docs/source/user-guide/cli/functions.md b/docs/source/user-guide/cli/functions.mdindex f3b0163534..11f61297ac 100644--- a/docs/source/user-guide/cli/functions.md+++ b/docs/source/user-guide/cli/functions.md@@ -170,5 +170,43 @@ The columns of the returned table are:| table_size_bytes | Utf8 | Size of the table, in bytes || statistics_size_bytes | UInt64 | Size of the cached statistics in memory |+## `list_files_cache`++The `list_files_cache` function shows information about the `ListFilesCache` that is used by the [`ListingTable`] implementation in DataFusion. When creating a [`ListingTable`], DataFusion lists the files in the table's location and caches results in the `ListFilesCache`. Subsequent queries against the same table can reuse this cached information instead of re-listing the files.++You can inspect the cache by querying the `list_files_cache` function. For example,++```sql+> select split_part(path, '/', -1) as folder, metadata_size_bytes, expires_in, unnest(metadata_list)['file_size_bytes'] as file_size_bytes, unnest(metadata_list)['e_tag'] as e_tag from list_files_cache();++----------+---------------------+-----------------------------------+-----------------+-------------------------------++| folder | metadata_size_bytes | expires_in | file_size_bytes | e_tag |++----------+---------------------+-----------------------------------+-----------------+-------------------------------++| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1233969 | 7041136-643a7bfeeec9b-12d431 |+| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1234756 | 7041137-643a7bfeef2df-12d744 |+| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1232554 | 7041139-643a7bfeef86a-12ceaa |+| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1238676 | 704113a-643a7bfeef914-12e694 |+| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1232186 | 704113b-643a7bfeefb22-12cd3a |+| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1237506 | 7041138-643a7bfeef775-12e202 |+| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1228756 | 7041134-643a7bfeec2d8-12bfd4 |+| customer | 1592 | 0 days 0 hours 0 mins 18.488 secs | 1228509 | 7041135-643a7bfeed599-12bedd |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20124715 | 704114a-643a7c00bb560-133142b |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20131024 | 7041149-643a7c00b90b7-1332cd0 |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20179217 | 704114b-643a7c00bb93e-133e911 |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20296819 | 704114f-643a7c00ccefd-135b473 |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20110730 | 7041148-643a7c00b9832-132dd8a |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20128346 | 704114c-643a7c00bc00a-133225a |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20130133 | 7041147-643a7c00b3901-1332955 |+| lineitem | 1600 | 0 days 0 hours 0 mins 16.758 secs | 20139830 | 7041146-643a7c00abbe8-1334f36 |++----------+---------------------+-----------------------------------+-----------------+-------------------------------++```++The columns of the returned table are:+| column_name | data_type | Description |+| ------------------- | ------------ | ----------------------------------------------------------------------------------------- |+| path | Utf8 | File path relative to the object store / filesystem root |+| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |+| expires_in | Duration(ms) | Last modified time of the file |+| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |+[`listingtable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html[entity tag]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag