Skip to content

Apache DataFusion SQL Query Engine

refactor: build the parquet stats future once before choosing whether to spawn it (#26952)

Andrew Lamb934691e9af8f
diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs
index baa28e9252..26e50d19a2 100644
--- a/datafusion/datasource-parquet/src/file_format.rs
+++ b/datafusion/datasource-parquet/src/file_format.rs
@@ -598,42 +598,33 @@ impl FileFormat for ParquetFormat {
// it in its own task to spread that work over the runtime's threads
// instead of decoding every file on the thread polling the planner.
// As in `spawn_buffered`, only spawn from a multi-thread tokio runtime.
+ let store = Arc::clone(store);
+ let object = object.clone();
+ let fetch = async move {
+ fetch_stats_and_ordering(
+ store.as_ref(),
+ &object,
+ metadata_size_hint,
+ file_decryption_properties,
+ file_metadata_cache,
+ &table_schema,
+ )
+ .await
+ };
match tokio::runtime::Handle::try_current() {
Ok(handle)
if handle.runtime_flavor()
== tokio::runtime::RuntimeFlavor::MultiThread =>
{
- let store = Arc::clone(store);
- let object = object.clone();
- SpawnedTask::spawn(async move {
- fetch_stats_and_ordering(
- store.as_ref(),
- &object,
- metadata_size_hint,
- file_decryption_properties,
- file_metadata_cache,
- &table_schema,
- )
+ SpawnedTask::spawn(fetch)
+ .join_unwind()
.await
- })
- .join_unwind()
- .await
- .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))?
+ .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))?
}
// Otherwise do the work on this task, where the caller polls it
// together with the other files (for example in `ListingTable`'s
// `buffer_unordered`).
- _ => {
- fetch_stats_and_ordering(
- store.as_ref(),
- object,
- metadata_size_hint,
- file_decryption_properties,
- file_metadata_cache,
- &table_schema,
- )
- .await
- }
+ _ => fetch.await,
}
}