Apache DataFusion SQL Query Engine
feat: Expose cache hits in list_files_cache function (#23439)
Michael Kleen65f9726f753b
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rsindex 7d87e7ed8a..164af2559d 100644--- a/datafusion-cli/src/functions.rs+++ b/datafusion-cli/src/functions.rs@@ -813,6 +813,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {DataType::List(Arc::new(metadata_field.clone())),true,),+ Field::new("hits", DataType::UInt64, false),]));let mut table_arr = vec![];@@ -826,6 +827,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {let mut etag_arr = vec![];let mut version_arr = vec![];let mut offsets: Vec<i32> = vec![0];+ let mut hits_arr = vec![];if let Some(list_files_cache) = self.cache_manager.get_list_files_cache() {let now = Instant::now();@@ -851,6 +853,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {}current_offset += entry.value.files.len() as i32;offsets.push(current_offset);+ hits_arr.push(entry.hits as u64);}}@@ -882,6 +885,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {Arc::new(struct_arr),None,)),+ Arc::new(UInt64Array::from(hits_arr)),],)?;diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rsindex 78d8342020..20a2537d7c 100644--- a/datafusion-cli/src/main.rs+++ b/datafusion-cli/src/main.rs@@ -810,7 +810,7 @@ mod tests {.collect().await?;- let sql = "SELECT metadata_size_bytes, expires_in, metadata_list FROM list_files_cache()";+ let sql = "SELECT metadata_size_bytes, expires_in, metadata_list, hits FROM list_files_cache()";let df = ctx.sql(sql).await?@@ -838,16 +838,17 @@ mod tests {"filename","file_size_bytes","etag",+ "hits",])?.sort(vec![col("filename").sort(true, false)])?;let rbs = df.collect().await?;assert_snapshot!(batches_to_string(&rbs),@r"- +---------------------+-----------+-----------------+------+- | metadata_size_bytes | filename | file_size_bytes | etag |- +---------------------+-----------+-----------------+------+- | 212 | 0.parquet | 3642 | 0 |- | 212 | 1.parquet | 3642 | 1 |- +---------------------+-----------+-----------------+------++ +---------------------+-----------+-----------------+------+------++ | metadata_size_bytes | filename | file_size_bytes | etag | hits |+ +---------------------+-----------+-----------------+------+------++ | 212 | 0.parquet | 3642 | 0 | 2 |+ | 212 | 1.parquet | 3642 | 1 | 2 |+ +---------------------+-----------+-----------------+------+------+");Ok(())diff --git a/docs/source/user-guide/cli/functions.md b/docs/source/user-guide/cli/functions.mdindex 409661ac82..baf054ef5a 100644--- a/docs/source/user-guide/cli/functions.md+++ b/docs/source/user-guide/cli/functions.md@@ -208,6 +208,7 @@ The columns of the returned table are:| path | Utf8 | File path relative to the object store / filesystem root || metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) || expires_in | Duration(ms) | Last modified time of the file |+| hits | UInt64 | Number of times the cached metadata has been accessed || metadata_list | List(Struct) | List of metadatas, one for each file under the path. |A metadata struct in the metadata_list contains the following fields: