Apache DataFusion SQL Query Engine
fix: isolate anonymous file statistics cache (#22950)
Kumar Ujjawal88273eb67def
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 74eb6ad39c..04feec2b6e 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -34,8 +34,7 @@ use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use datafusion_datasource::{ListingTableUrl, PartitionedFile, TableSchemaBuilder, compute_all_files_statistics,};-use datafusion_execution::cache::cache_manager::FileStatisticsCache;-use datafusion_execution::cache::cache_manager::TableScopedPath;+use datafusion_execution::cache::cache_manager::{FileStatisticsCache, TableScopedPath};use datafusion_expr::dml::InsertOp;use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType};@@ -264,6 +263,21 @@ impl ListingTable {self}+ fn statistics_cache(+ &self,+ has_table_reference: bool,+ ) -> Option<&Arc<FileStatisticsCache>> {+ let shared_cache = self.collected_statistics.as_ref()?;+ if has_table_reference || self.schema_source == SchemaSource::Inferred {+ Some(shared_cache)+ } else {+ // Anonymous specified-schema reads can use the same file path with+ // different logical schemas. File statistics are schema-dependent,+ // so avoid reusing stats computed for a different read schema.+ None+ }+ }+/// Specify the SQL definition for this table, if anypub fn with_definition(mut self, definition: Option<String>) -> Self {self.definition = definition;@@ -807,7 +821,7 @@ impl ListingTable {let meta = &part_file.object_meta;// Check cache first - if we have valid cached statistics and ordering- if let Some(cache) = &self.collected_statistics+ if let Some(cache) = self.statistics_cache(path.table.is_some())&& let Some(cached) = cache.get(&path)&& cached.is_valid_for(meta){@@ -825,7 +839,7 @@ impl ListingTable {let statistics = Arc::new(file_meta.statistics);// Store in cache- if let Some(cache) = &self.collected_statistics {+ if let Some(cache) = self.statistics_cache(path.table.is_some()) {cache.put(&path,CachedFileMetadata::new(diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rsindex f907d715b8..8ef780ea1e 100644--- a/datafusion/core/src/datasource/file_format/options.rs+++ b/datafusion/core/src/datasource/file_format/options.rs@@ -34,6 +34,7 @@ use crate::error::Result;use crate::execution::context::{SessionConfig, SessionState};use arrow::datatypes::{DataType, Schema, SchemaRef};+use datafusion_catalog_listing::SchemaSource;use datafusion_common::config::{ConfigFileDecryptionProperties, TableOptions};use datafusion_common::{DEFAULT_ARROW_EXTENSION, DEFAULT_AVRO_EXTENSION, DEFAULT_CSV_EXTENSION,@@ -595,6 +596,11 @@ pub trait ReadOptions<'a> {table_path: ListingTableUrl,) -> Result<SchemaRef>;+ /// Returns whether the read schema was inferred or specified.+ fn schema_source(&self) -> SchemaSource {+ SchemaSource::Specified+ }+/// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.async fn _get_resolved_schema(&'a self,@@ -652,6 +658,10 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {self._get_resolved_schema(config, state, table_path, self.schema).await}++ fn schema_source(&self) -> SchemaSource {+ schema_source_from_option(self.schema)+ }}#[cfg(feature = "parquet")]@@ -695,6 +705,10 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {self._get_resolved_schema(config, state, table_path, self.schema).await}++ fn schema_source(&self) -> SchemaSource {+ schema_source_from_option(self.schema)+ }}#[async_trait]@@ -725,6 +739,10 @@ impl ReadOptions<'_> for JsonReadOptions<'_> {self._get_resolved_schema(config, state, table_path, self.schema).await}++ fn schema_source(&self) -> SchemaSource {+ schema_source_from_option(self.schema)+ }}#[cfg(feature = "avro")]@@ -751,6 +769,10 @@ impl ReadOptions<'_> for AvroReadOptions<'_> {self._get_resolved_schema(config, state, table_path, self.schema).await}++ fn schema_source(&self) -> SchemaSource {+ schema_source_from_option(self.schema)+ }}#[async_trait]@@ -776,4 +798,16 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> {self._get_resolved_schema(config, state, table_path, self.schema).await}++ fn schema_source(&self) -> SchemaSource {+ schema_source_from_option(self.schema)+ }+}++fn schema_source_from_option(schema: Option<&Schema>) -> SchemaSource {+ if schema.is_some() {+ SchemaSource::Specified+ } else {+ SchemaSource::Inferred+ }}diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 0532e34fbc..73847b67ed 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -65,6 +65,7 @@ use datafusion_catalog::memory::MemorySchemaProvider;use datafusion_catalog::{DynamicFileCatalog, TableFunction, TableFunctionImpl, UrlTableFactory,};+use datafusion_catalog_listing::SchemaSource;use datafusion_common::config::{ConfigField, ConfigOptions};use datafusion_common::metadata::ScalarAndMetadata;use datafusion_common::{@@ -1725,12 +1726,20 @@ impl SessionContext {}}- let resolved_schema = options- .get_resolved_schema(&session_config, self.state(), table_paths[0].clone())- .await?;+ let schema_table_path = table_paths[0].clone();let config = ListingTableConfig::new_with_multi_paths(table_paths)- .with_listing_options(listing_options)- .with_schema(resolved_schema);+ .with_listing_options(listing_options);+ let config = match options.schema_source() {+ SchemaSource::Inferred | SchemaSource::Unset => {+ config.infer_schema(&self.state()).await?+ }+ SchemaSource::Specified => {+ let resolved_schema = options+ .get_resolved_schema(&session_config, self.state(), schema_table_path)+ .await?;+ config.with_schema(resolved_schema)+ }+ };let provider = ListingTable::try_new(config)?.with_cache(self.runtime_env().cache_manager.get_file_statistic_cache());self.read_table(Arc::new(provider))diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rsindex 45c0b66a6c..b082271d67 100644--- a/datafusion/core/tests/parquet/file_statistics.rs+++ b/datafusion/core/tests/parquet/file_statistics.rs@@ -18,6 +18,7 @@use std::fs;use std::sync::Arc;+use arrow::datatypes::{DataType, Field, Schema};use datafusion::datasource::TableProvider;use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::{@@ -26,9 +27,9 @@ use datafusion::datasource::listing::{use datafusion::datasource::source::DataSourceExec;use datafusion::execution::context::SessionState;use datafusion::execution::session_state::SessionStateBuilder;-use datafusion::prelude::SessionContext;-use datafusion_common::DFSchema;+use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion_common::stats::Precision;+use datafusion_common::{DFSchema, TableReference};use datafusion_execution::cache::cache_manager::{CacheManagerConfig, DEFAULT_FILE_STATISTICS_MEMORY_LIMIT,DEFAULT_LIST_FILES_CACHE_MEMORY_LIMIT, FileStatisticsCache, ListFilesCache,@@ -111,7 +112,9 @@ async fn check_stats_precision_with_filter_pushdown() {async fn load_table_stats_with_session_level_cache() {let testdata = datafusion::test_util::parquet_test_data();let filename = format!("{}/{}", testdata, "alltypes_plain.parquet");- let table_path = ListingTableUrl::parse(filename).unwrap();+ let table_path = ListingTableUrl::parse(filename)+ .unwrap()+ .with_table_ref(TableReference::bare("alltypes_plain"));let (cache1, _, mut state1) = get_cache_runtime_state();let cfg_1 = state1.config_mut();@@ -193,6 +196,68 @@ async fn load_table_stats_with_session_level_cache() {assert_eq!(get_static_cache_size(&state1), 1);}+#[tokio::test]+async fn anonymous_parquet_stats_cache_with_explicit_wider_schema() {+ let temp_dir = tempdir().unwrap();+ let parquet_path = temp_dir.path().join("data.parquet");+ let parquet_path = parquet_path.to_string_lossy().to_string();++ let ctx = SessionContext::new_with_config(+ SessionConfig::new().with_collect_statistics(true),+ );+ let cache = ctx+ .runtime_env()+ .cache_manager+ .get_file_statistic_cache()+ .unwrap();++ ctx.sql(&format!(+ "COPY (+ SELECT 1::BIGINT AS id, 1000::BIGINT AS population+ ) TO '{parquet_path}' STORED AS PARQUET"+ ))+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();++ assert_eq!(cache.len(), 0);++ ctx.read_parquet(&parquet_path, ParquetReadOptions::default())+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();++ assert_eq!(cache.len(), 1);++ let wider_schema = Schema::new(vec![+ Field::new("id", DataType::Int64, true),+ Field::new("population", DataType::Int64, true),+ Field::new("extra", DataType::Int64, true),+ ]);++ let plan = ctx+ .read_parquet(+ &parquet_path,+ ParquetReadOptions::default().schema(&wider_schema),+ )+ .await+ .unwrap()+ .select_columns(&["id", "extra"])+ .unwrap()+ .create_physical_plan()+ .await+ .unwrap();++ let stats = plan.statistics_with_args(&StatisticsArgs::new()).unwrap();+ assert_eq!(stats.column_statistics.len(), 2);+ assert_eq!(stats.column_statistics[1].null_count, Precision::Exact(1));+ assert_eq!(cache.len(), 1);+}+#[tokio::test]async fn list_files_with_session_level_cache() {let p_name = "alltypes_plain.parquet";