Skip to content

Apache DataFusion SQL Query Engine

perf: decode Parquet footers for statistics in parallel during planning (#26407)

Peter La0330ea3ecd6
diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs
index b62feac2d8..baa28e9252 100644
--- a/datafusion/datasource-parquet/src/file_format.rs
+++ b/datafusion/datasource-parquet/src/file_format.rs
@@ -38,10 +38,12 @@ use datafusion_common::{
DEFAULT_PARQUET_EXTENSION, DataFusionError, GetExt, Result, internal_datafusion_err,
internal_err, not_impl_err,
};
+use datafusion_common_runtime::SpawnedTask;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
use datafusion_datasource::sink::DataSinkExec;
use datafusion_datasource::write::get_writer_schema;
+use datafusion_execution::cache::cache_manager::FileMetadataCache;
use datafusion_expr::dml::InsertOp;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};
use datafusion_physical_plan::ExecutionPlan;
@@ -388,6 +390,32 @@ async fn get_file_decryption_properties(
Ok(None)
}
+/// Fetch the footer of `object` and compute its statistics and ordering for
+/// [`ParquetFormat::infer_stats_and_ordering`].
+async fn fetch_stats_and_ordering(
+ store: &dyn ObjectStore,
+ object: &ObjectMeta,
+ metadata_size_hint: Option<usize>,
+ file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
+ file_metadata_cache: Arc<FileMetadataCache>,
+ table_schema: &SchemaRef,
+) -> Result<datafusion_datasource::file_format::FileMeta> {
+ let metadata = DFParquetMetadata::new(store, object)
+ .with_metadata_size_hint(metadata_size_hint)
+ .with_decryption_properties(file_decryption_properties)
+ .with_file_metadata_cache(Some(file_metadata_cache))
+ .fetch_metadata()
+ .await?;
+ let statistics =
+ DFParquetMetadata::statistics_from_parquet_metadata(&metadata, table_schema)?;
+ let ordering =
+ crate::metadata::ordering_from_parquet_metadata(&metadata, table_schema)?;
+ Ok(
+ datafusion_datasource::file_format::FileMeta::new(statistics)
+ .with_ordering(ordering),
+ )
+}
+
#[async_trait]
impl FileFormat for ParquetFormat {
fn get_ext(&self) -> String {
@@ -562,22 +590,51 @@ impl FileFormat for ParquetFormat {
.await?;
let file_metadata_cache =
state.runtime_env().cache_manager.get_file_metadata_cache();
- let metadata = DFParquetMetadata::new(store, object)
- .with_metadata_size_hint(self.metadata_size_hint())
- .with_decryption_properties(file_decryption_properties)
- .with_file_metadata_cache(Some(file_metadata_cache))
- .fetch_metadata()
- .await?;
- let statistics = DFParquetMetadata::statistics_from_parquet_metadata(
- &metadata,
- &table_schema,
- )?;
- let ordering =
- crate::metadata::ordering_from_parquet_metadata(&metadata, &table_schema)?;
- Ok(
- datafusion_datasource::file_format::FileMeta::new(statistics)
- .with_ordering(ordering),
- )
+ let metadata_size_hint = self.metadata_size_hint();
+
+ // Decoding the footer and converting its statistics is CPU bound and
+ // grows with the number of columns. Callers such as `ListingTable` run
+ // this for many files concurrently from a single planning task, so run
+ // it in its own task to spread that work over the runtime's threads
+ // instead of decoding every file on the thread polling the planner.
+ // As in `spawn_buffered`, only spawn from a multi-thread tokio runtime.
+ match tokio::runtime::Handle::try_current() {
+ Ok(handle)
+ if handle.runtime_flavor()
+ == tokio::runtime::RuntimeFlavor::MultiThread =>
+ {
+ let store = Arc::clone(store);
+ let object = object.clone();
+ SpawnedTask::spawn(async move {
+ fetch_stats_and_ordering(
+ store.as_ref(),
+ &object,
+ metadata_size_hint,
+ file_decryption_properties,
+ file_metadata_cache,
+ &table_schema,
+ )
+ .await
+ })
+ .join_unwind()
+ .await
+ .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))?
+ }
+ // Otherwise do the work on this task, where the caller polls it
+ // together with the other files (for example in `ListingTable`'s
+ // `buffer_unordered`).
+ _ => {
+ fetch_stats_and_ordering(
+ store.as_ref(),
+ object,
+ metadata_size_hint,
+ file_decryption_properties,
+ file_metadata_cache,
+ &table_schema,
+ )
+ .await
+ }
+ }
}
async fn create_physical_plan(