Skip to content

Apache DataFusion SQL Query Engine

fix: isolate anonymous file statistics cache (#22950)

Kumar Ujjawal88273eb67def
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs
index 74eb6ad39c..04feec2b6e 100644
--- a/datafusion/catalog-listing/src/table.rs
+++ b/datafusion/catalog-listing/src/table.rs
@@ -34,8 +34,7 @@ use datafusion_datasource::schema_adapter::SchemaAdapterFactory;
use datafusion_datasource::{
ListingTableUrl, PartitionedFile, TableSchemaBuilder, compute_all_files_statistics,
};
-use datafusion_execution::cache::cache_manager::FileStatisticsCache;
-use datafusion_execution::cache::cache_manager::TableScopedPath;
+use datafusion_execution::cache::cache_manager::{FileStatisticsCache, TableScopedPath};
use datafusion_expr::dml::InsertOp;
use datafusion_expr::execution_props::ExecutionProps;
use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType};
@@ -264,6 +263,21 @@ impl ListingTable {
self
}
+ fn statistics_cache(
+ &self,
+ has_table_reference: bool,
+ ) -> Option<&Arc<FileStatisticsCache>> {
+ let shared_cache = self.collected_statistics.as_ref()?;
+ if has_table_reference || self.schema_source == SchemaSource::Inferred {
+ Some(shared_cache)
+ } else {
+ // Anonymous specified-schema reads can use the same file path with
+ // different logical schemas. File statistics are schema-dependent,
+ // so avoid reusing stats computed for a different read schema.
+ None
+ }
+ }
+
/// Specify the SQL definition for this table, if any
pub fn with_definition(mut self, definition: Option<String>) -> Self {
self.definition = definition;
@@ -807,7 +821,7 @@ impl ListingTable {
let meta = &part_file.object_meta;
// Check cache first - if we have valid cached statistics and ordering
- if let Some(cache) = &self.collected_statistics
+ if let Some(cache) = self.statistics_cache(path.table.is_some())
&& let Some(cached) = cache.get(&path)
&& cached.is_valid_for(meta)
{
@@ -825,7 +839,7 @@ impl ListingTable {
let statistics = Arc::new(file_meta.statistics);
// Store in cache
- if let Some(cache) = &self.collected_statistics {
+ if let Some(cache) = self.statistics_cache(path.table.is_some()) {
cache.put(
&path,
CachedFileMetadata::new(
diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs
index f907d715b8..8ef780ea1e 100644
--- a/datafusion/core/src/datasource/file_format/options.rs
+++ b/datafusion/core/src/datasource/file_format/options.rs
@@ -34,6 +34,7 @@ use crate::error::Result;
use crate::execution::context::{SessionConfig, SessionState};
use arrow::datatypes::{DataType, Schema, SchemaRef};
+use datafusion_catalog_listing::SchemaSource;
use datafusion_common::config::{ConfigFileDecryptionProperties, TableOptions};
use datafusion_common::{
DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION,
@@ -595,6 +596,11 @@ pub trait ReadOptions<'a> {
table_path: ListingTableUrl,
) -> Result<SchemaRef>;
+ /// Returns whether the read schema was inferred or specified.
+ fn schema_source(&self) -> SchemaSource {
+ SchemaSource::Specified
+ }
+
/// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.
async fn _get_resolved_schema(
&'a self,
@@ -652,6 +658,10 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {
self._get_resolved_schema(config, state, table_path, self.schema)
.await
}
+
+ fn schema_source(&self) -> SchemaSource {
+ schema_source_from_option(self.schema)
+ }
}
#[cfg(feature = "parquet")]
@@ -695,6 +705,10 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {
self._get_resolved_schema(config, state, table_path, self.schema)
.await
}
+
+ fn schema_source(&self) -> SchemaSource {
+ schema_source_from_option(self.schema)
+ }
}
#[async_trait]
@@ -725,6 +739,10 @@ impl ReadOptions<'_> for JsonReadOptions<'_> {
self._get_resolved_schema(config, state, table_path, self.schema)
.await
}
+
+ fn schema_source(&self) -> SchemaSource {
+ schema_source_from_option(self.schema)
+ }
}
#[cfg(feature = "avro")]
@@ -751,6 +769,10 @@ impl ReadOptions<'_> for AvroReadOptions<'_> {
self._get_resolved_schema(config, state, table_path, self.schema)
.await
}
+
+ fn schema_source(&self) -> SchemaSource {
+ schema_source_from_option(self.schema)
+ }
}
#[async_trait]
@@ -776,4 +798,16 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> {
self._get_resolved_schema(config, state, table_path, self.schema)
.await
}
+
+ fn schema_source(&self) -> SchemaSource {
+ schema_source_from_option(self.schema)
+ }
+}
+
+fn schema_source_from_option(schema: Option<&Schema>) -> SchemaSource {
+ if schema.is_some() {
+ SchemaSource::Specified
+ } else {
+ SchemaSource::Inferred
+ }
}
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs
index 0532e34fbc..73847b67ed 100644
--- a/datafusion/core/src/execution/context/mod.rs
+++ b/datafusion/core/src/execution/context/mod.rs
@@ -65,6 +65,7 @@ use datafusion_catalog::memory::MemorySchemaProvider;
use datafusion_catalog::{
DynamicFileCatalog, TableFunction, TableFunctionImpl, UrlTableFactory,
};
+use datafusion_catalog_listing::SchemaSource;
use datafusion_common::config::{ConfigField, ConfigOptions};
use datafusion_common::metadata::ScalarAndMetadata;
use datafusion_common::{
@@ -1725,12 +1726,20 @@ impl SessionContext {
}
}
- let resolved_schema = options
- .get_resolved_schema(&session_config, self.state(), table_paths[0].clone())
- .await?;
+ let schema_table_path = table_paths[0].clone();
let config = ListingTableConfig::new_with_multi_paths(table_paths)
- .with_listing_options(listing_options)
- .with_schema(resolved_schema);
+ .with_listing_options(listing_options);
+ let config = match options.schema_source() {
+ SchemaSource::Inferred | SchemaSource::Unset => {
+ config.infer_schema(&self.state()).await?
+ }
+ SchemaSource::Specified => {
+ let resolved_schema = options
+ .get_resolved_schema(&session_config, self.state(), schema_table_path)
+ .await?;
+ config.with_schema(resolved_schema)
+ }
+ };
let provider = ListingTable::try_new(config)?
.with_cache(self.runtime_env().cache_manager.get_file_statistic_cache());
self.read_table(Arc::new(provider))
diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rs
index 45c0b66a6c..b082271d67 100644
--- a/datafusion/core/tests/parquet/file_statistics.rs
+++ b/datafusion/core/tests/parquet/file_statistics.rs
@@ -18,6 +18,7 @@
use std::fs;
use std::sync::Arc;
+use arrow::datatypes::{DataType, Field, Schema};
use datafusion::datasource::TableProvider;
use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::{
@@ -26,9 +27,9 @@ use datafusion::datasource::listing::{
use datafusion::datasource::source::DataSourceExec;
use datafusion::execution::context::SessionState;
use datafusion::execution::session_state::SessionStateBuilder;
-use datafusion::prelude::SessionContext;
-use datafusion_common::DFSchema;
+use datafusion::prelude::{ParquetReadOptions, SessionContext};
use datafusion_common::stats::Precision;
+use datafusion_common::{DFSchema, TableReference};
use datafusion_execution::cache::cache_manager::{
CacheManagerConfig, DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,
DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, FileStatisticsCache, ListFilesCache,
@@ -111,7 +112,9 @@ async fn check_stats_precision_with_filter_pushdown() {
async fn load_table_stats_with_session_level_cache() {
let testdata = datafusion::test_util::parquet_test_data();
let filename = format!("{}/{}", testdata, "alltypes_plain.parquet");
- let table_path = ListingTableUrl::parse(filename).unwrap();
+ let table_path = ListingTableUrl::parse(filename)
+ .unwrap()
+ .with_table_ref(TableReference::bare("alltypes_plain"));
let (cache1, _, mut state1) = get_cache_runtime_state();
let cfg_1 = state1.config_mut();
@@ -193,6 +196,68 @@ async fn load_table_stats_with_session_level_cache() {
assert_eq!(get_static_cache_size(&state1), 1);
}
+#[tokio::test]
+async fn anonymous_parquet_stats_cache_with_explicit_wider_schema() {
+ let temp_dir = tempdir().unwrap();
+ let parquet_path = temp_dir.path().join("data.parquet");
+ let parquet_path = parquet_path.to_string_lossy().to_string();
+
+ let ctx = SessionContext::new_with_config(
+ SessionConfig::new().with_collect_statistics(true),
+ );
+ let cache = ctx
+ .runtime_env()
+ .cache_manager
+ .get_file_statistic_cache()
+ .unwrap();
+
+ ctx.sql(&format!(
+ "COPY (
+ SELECT 1::BIGINT AS id, 1000::BIGINT AS population
+ ) TO '{parquet_path}' STORED AS PARQUET"
+ ))
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+
+ assert_eq!(cache.len(), 0);
+
+ ctx.read_parquet(&parquet_path, ParquetReadOptions::default())
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+
+ assert_eq!(cache.len(), 1);
+
+ let wider_schema = Schema::new(vec![
+ Field::new("id", DataType::Int64, true),
+ Field::new("population", DataType::Int64, true),
+ Field::new("extra", DataType::Int64, true),
+ ]);
+
+ let plan = ctx
+ .read_parquet(
+ &parquet_path,
+ ParquetReadOptions::default().schema(&wider_schema),
+ )
+ .await
+ .unwrap()
+ .select_columns(&["id", "extra"])
+ .unwrap()
+ .create_physical_plan()
+ .await
+ .unwrap();
+
+ let stats = plan.statistics_with_args(&StatisticsArgs::new()).unwrap();
+ assert_eq!(stats.column_statistics.len(), 2);
+ assert_eq!(stats.column_statistics[1].null_count, Precision::Exact(1));
+ assert_eq!(cache.len(), 1);
+}
+
#[tokio::test]
async fn list_files_with_session_level_cache() {
let p_name = "alltypes_plain.parquet";