Skip to content

Apache DataFusion SQL Query Engine

feat: Expose cache hits in statistics_cache function (#23253)

Michael Kleen4d898e93f09f
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs
index cb2372958e..7d87e7ed8a 100644
--- a/datafusion-cli/src/functions.rs
+++ b/datafusion-cli/src/functions.rs
@@ -649,6 +649,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
Field::new("num_columns", DataType::UInt64, false),
Field::new("table_size_bytes", DataType::Utf8, false),
Field::new("statistics_size_bytes", DataType::UInt64, false),
+ Field::new("hits", DataType::UInt64, false),
]));
// construct record batch from metadata
@@ -662,6 +663,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
let mut num_columns_arr = vec![];
let mut table_size_bytes_arr = vec![];
let mut statistics_size_bytes_arr = vec![];
+ let mut hits_arr = vec![];
if let Some(file_statistics_cache) = self.cache_manager.get_file_statistic_cache()
{
@@ -686,6 +688,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
.heap_size(&mut DFHeapSizeCtx::default())
as u64,
);
+ hits_arr.push(entry.hits as u64);
}
}
@@ -702,6 +705,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
Arc::new(UInt64Array::from(num_columns_arr)),
Arc::new(StringArray::from(table_size_bytes_arr)),
Arc::new(UInt64Array::from(statistics_size_bytes_arr)),
+ Arc::new(UInt64Array::from(hits_arr)),
],
)?;
diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
index cf77e5415d..78d8342020 100644
--- a/datafusion-cli/src/main.rs
+++ b/datafusion-cli/src/main.rs
@@ -711,17 +711,48 @@ mod tests {
.await?;
}
- let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, table_size_bytes from statistics_cache() order by filename";
+ let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";
let df = ctx.sql(sql).await?;
let rbs = df.collect().await?;
- assert_snapshot!(batches_to_string(&rbs),@r"
- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
- | filename | table | file_size_bytes | num_rows | num_columns | table_size_bytes |
- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
- | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | Absent |
- | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | Absent |
- | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | Absent |
- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
+ assert_snapshot!(batches_to_string(&rbs),@"
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 0 | Absent |
+ | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |
+ | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 0 | Absent |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ ");
+
+ // increase the number of hits
+ ctx.sql("select * from alltypes_plain")
+ .await?
+ .collect()
+ .await?;
+ ctx.sql("select * from alltypes_plain")
+ .await?
+ .collect()
+ .await?;
+ ctx.sql("select * from alltypes_plain")
+ .await?
+ .collect()
+ .await?;
+ ctx.sql("select * from lz4_raw_compressed_larger")
+ .await?
+ .collect()
+ .await?;
+
+ let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";
+ let df = ctx.sql(sql).await?;
+ let rbs = df.collect().await?;
+ assert_snapshot!(batches_to_string(&rbs),@"
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 3 | Absent |
+ | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |
+ | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 1 | Absent |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
");
Ok(())
diff --git a/docs/source/user-guide/cli/functions.md b/docs/source/user-guide/cli/functions.md
index ea353d5c8d..409661ac82 100644
--- a/docs/source/user-guide/cli/functions.md
+++ b/docs/source/user-guide/cli/functions.md
@@ -168,6 +168,7 @@ The columns of the returned table are:
| num_rows | Utf8 | Number of rows in the table |
| num_columns | UInt64 | Number of columns in the table |
| table_size_bytes | Utf8 | Size of the table, in bytes |
+| hits | UInt64 | Number of times the cached file statistics has been accessed |
| statistics_size_bytes | UInt64 | Size of the cached statistics in memory |
## `list_files_cache`
@@ -200,13 +201,14 @@ location 's3://overturemaps-us-west-2/release/2025-12-17.0/theme=base/type=infra
```
The columns of the returned table are:
-| column_name | data_type | Description |
-| ------------------- | ------------ | ----------------------------------------------------------------------------------------- |
-| table | Utf8 | Name of the table |
-| path | Utf8 | File path relative to the object store / filesystem root |
-| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |
-| expires_in | Duration(ms) | Last modified time of the file |
-| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
+
+| column_name | data_type | Description |
+| ------------------- | ------------ | ------------------------------------------------------------------- |
+| table | Utf8 | Name of the table |
+| path | Utf8 | File path relative to the object store / filesystem root |
+| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |
+| expires_in | Duration(ms) | Last modified time of the file |
+| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
A metadata struct in the metadata_list contains the following fields: