Skip to content

Apache DataFusion SQL Query Engine

feat: Expose cache hits in list_files_cache function (#23439)

Michael Kleen65f9726f753b
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs
index 7d87e7ed8a..164af2559d 100644
--- a/datafusion-cli/src/functions.rs
+++ b/datafusion-cli/src/functions.rs
@@ -813,6 +813,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
DataType::List(Arc::new(metadata_field.clone())),
true,
),
+ Field::new("hits", DataType::UInt64, false),
]));
let mut table_arr = vec![];
@@ -826,6 +827,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
let mut etag_arr = vec![];
let mut version_arr = vec![];
let mut offsets: Vec<i32> = vec![0];
+ let mut hits_arr = vec![];
if let Some(list_files_cache) = self.cache_manager.get_list_files_cache() {
let now = Instant::now();
@@ -851,6 +853,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
}
current_offset += entry.value.files.len() as i32;
offsets.push(current_offset);
+ hits_arr.push(entry.hits as u64);
}
}
@@ -882,6 +885,7 @@ impl TableFunctionImpl for ListFilesCacheFunc {
Arc::new(struct_arr),
None,
)),
+ Arc::new(UInt64Array::from(hits_arr)),
],
)?;
diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
index 78d8342020..20a2537d7c 100644
--- a/datafusion-cli/src/main.rs
+++ b/datafusion-cli/src/main.rs
@@ -810,7 +810,7 @@ mod tests {
.collect()
.await?;
- let sql = "SELECT metadata_size_bytes, expires_in, metadata_list FROM list_files_cache()";
+ let sql = "SELECT metadata_size_bytes, expires_in, metadata_list, hits FROM list_files_cache()";
let df = ctx
.sql(sql)
.await?
@@ -838,16 +838,17 @@ mod tests {
"filename",
"file_size_bytes",
"etag",
+ "hits",
])?
.sort(vec![col("filename").sort(true, false)])?;
let rbs = df.collect().await?;
assert_snapshot!(batches_to_string(&rbs),@r"
- +---------------------+-----------+-----------------+------+
- | metadata_size_bytes | filename | file_size_bytes | etag |
- +---------------------+-----------+-----------------+------+
- | 212 | 0.parquet | 3642 | 0 |
- | 212 | 1.parquet | 3642 | 1 |
- +---------------------+-----------+-----------------+------+
+ +---------------------+-----------+-----------------+------+------+
+ | metadata_size_bytes | filename | file_size_bytes | etag | hits |
+ +---------------------+-----------+-----------------+------+------+
+ | 212 | 0.parquet | 3642 | 0 | 2 |
+ | 212 | 1.parquet | 3642 | 1 | 2 |
+ +---------------------+-----------+-----------------+------+------+
");
Ok(())
diff --git a/docs/source/user-guide/cli/functions.md b/docs/source/user-guide/cli/functions.md
index 409661ac82..baf054ef5a 100644
--- a/docs/source/user-guide/cli/functions.md
+++ b/docs/source/user-guide/cli/functions.md
@@ -208,6 +208,7 @@ The columns of the returned table are:
| path | Utf8 | File path relative to the object store / filesystem root |
| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |
| expires_in | Duration(ms) | Last modified time of the file |
+| hits | UInt64 | Number of times the cached metadata has been accessed |
| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
A metadata struct in the metadata_list contains the following fields: