Apache DataFusion SQL Query Engine
feat: Expose cache hits in statistics_cache function (#23253)
Michael Kleen4d898e93f09f
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rsindex cb2372958e..7d87e7ed8a 100644--- a/datafusion-cli/src/functions.rs+++ b/datafusion-cli/src/functions.rs@@ -649,6 +649,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {Field::new("num_columns", DataType::UInt64, false),Field::new("table_size_bytes", DataType::Utf8, false),Field::new("statistics_size_bytes", DataType::UInt64, false),+ Field::new("hits", DataType::UInt64, false),]));// construct record batch from metadata@@ -662,6 +663,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {let mut num_columns_arr = vec![];let mut table_size_bytes_arr = vec![];let mut statistics_size_bytes_arr = vec![];+ let mut hits_arr = vec![];if let Some(file_statistics_cache) = self.cache_manager.get_file_statistic_cache(){@@ -686,6 +688,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {.heap_size(&mut DFHeapSizeCtx::default())as u64,);+ hits_arr.push(entry.hits as u64);}}@@ -702,6 +705,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {Arc::new(UInt64Array::from(num_columns_arr)),Arc::new(StringArray::from(table_size_bytes_arr)),Arc::new(UInt64Array::from(statistics_size_bytes_arr)),+ Arc::new(UInt64Array::from(hits_arr)),],)?;diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rsindex cf77e5415d..78d8342020 100644--- a/datafusion-cli/src/main.rs+++ b/datafusion-cli/src/main.rs@@ -711,17 +711,48 @@ mod tests {.await?;}- let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, table_size_bytes from statistics_cache() order by filename";+ let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";let df = ctx.sql(sql).await?;let rbs = df.collect().await?;- assert_snapshot!(batches_to_string(&rbs),@r"- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+- | filename | table | file_size_bytes | num_rows | num_columns | table_size_bytes |- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+- | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | Absent |- | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | Absent |- | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | Absent |- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------++ assert_snapshot!(batches_to_string(&rbs),@"+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------++ | filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------++ | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 0 | Absent |+ | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |+ | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 0 | Absent |+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------++ ");++ // increase the number of hits+ ctx.sql("select * from alltypes_plain")+ .await?+ .collect()+ .await?;+ ctx.sql("select * from alltypes_plain")+ .await?+ .collect()+ .await?;+ ctx.sql("select * from alltypes_plain")+ .await?+ .collect()+ .await?;+ ctx.sql("select * from lz4_raw_compressed_larger")+ .await?+ .collect()+ .await?;++ let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";+ let df = ctx.sql(sql).await?;+ let rbs = df.collect().await?;+ assert_snapshot!(batches_to_string(&rbs),@"+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------++ | filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------++ | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 3 | Absent |+ | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |+ | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 1 | Absent |+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+");Ok(())diff --git a/docs/source/user-guide/cli/functions.md b/docs/source/user-guide/cli/functions.mdindex ea353d5c8d..409661ac82 100644--- a/docs/source/user-guide/cli/functions.md+++ b/docs/source/user-guide/cli/functions.md@@ -168,6 +168,7 @@ The columns of the returned table are:| num_rows | Utf8 | Number of rows in the table || num_columns | UInt64 | Number of columns in the table || table_size_bytes | Utf8 | Size of the table, in bytes |+| hits | UInt64 | Number of times the cached file statistics has been accessed || statistics_size_bytes | UInt64 | Size of the cached statistics in memory |## `list_files_cache`@@ -200,13 +201,14 @@ location 's3://overturemaps-us-west-2/release/2025-12-17.0/theme=base/type=infra```The columns of the returned table are:-| column_name | data_type | Description |-| ------------------- | ------------ | ----------------------------------------------------------------------------------------- |-| table | Utf8 | Name of the table |-| path | Utf8 | File path relative to the object store / filesystem root |-| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |-| expires_in | Duration(ms) | Last modified time of the file |-| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |++| column_name | data_type | Description |+| ------------------- | ------------ | ------------------------------------------------------------------- |+| table | Utf8 | Name of the table |+| path | Utf8 | File path relative to the object store / filesystem root |+| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |+| expires_in | Duration(ms) | Last modified time of the file |+| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |A metadata struct in the metadata_list contains the following fields: