Apache DataFusion SQL Query Engine
perf(core): cut `datafusion` core compile time ~10x (#24329)
Daniël Heresca1e25e23557
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 325ae91d27..8be23f927e 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -70,6 +70,7 @@ use datafusion_functions_aggregate::expr_fn::{use async_trait::async_trait;use datafusion_catalog::Session;use datafusion_expr::extension_types::DFArrayFormatterFactory;+use futures::future::BoxFuture;/// Contains options that control how data is/// written out from a DataFrame@@ -2716,7 +2717,36 @@ impl TableProvider for DataFrameTableProvider {self.table_type}- async fn scan(+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ projection: Option<&'life2 Vec<usize>>,+ filters: &'life3 [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ 'life3: 'async_trait,+ Self: 'async_trait,+ {+ self.scan_boxed(state, projection, filters, limit)+ }+}++impl DataFrameTableProvider {+ fn scan_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ projection: Option<&'a Vec<usize>>,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.scan_inner(state, projection, filters, limit))+ }++ async fn scan_inner(&self,state: &dyn Session,projection: Option<&Vec<usize>>,diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rsindex 50ee96da3d..f670c9cc44 100644--- a/datafusion/core/src/datasource/dynamic_file.rs+++ b/datafusion/core/src/datasource/dynamic_file.rs@@ -31,6 +31,7 @@ use datafusion_common::plan_datafusion_err;use datafusion_session::SessionStore;use async_trait::async_trait;+use futures::future::BoxFuture;/// [DynamicListTableFactory] is a factory that can create a [ListingTable] from the given url.#[derive(Default, Debug)]@@ -53,7 +54,28 @@ impl DynamicListTableFactory {#[async_trait]impl UrlTableFactory for DynamicListTableFactory {- async fn try_new(&self, url: &str) -> Result<Option<Arc<dyn TableProvider>>> {+ fn try_new<'life0, 'life1, 'async_trait>(+ &'life0 self,+ url: &'life1 str,+ ) -> BoxFuture<'async_trait, Result<Option<Arc<dyn TableProvider>>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ self.try_new_boxed(url)+ }+}++impl DynamicListTableFactory {+ fn try_new_boxed<'a>(+ &'a self,+ url: &'a str,+ ) -> BoxFuture<'a, Result<Option<Arc<dyn TableProvider>>>> {+ Box::pin(self.try_new_inner(url))+ }++ async fn try_new_inner(&self, url: &str) -> Result<Option<Arc<dyn TableProvider>>> {let Ok(table_url) = ListingTableUrl::parse(url) else {return Ok(None);};diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rsindex 8ef780ea1e..4babf4a421 100644--- a/datafusion/core/src/datasource/file_format/options.rs+++ b/datafusion/core/src/datasource/file_format/options.rs@@ -17,6 +17,7 @@//! User facing options for the file formats readers+use std::future::ready;use std::sync::Arc;#[cfg(feature = "avro")]@@ -44,6 +45,7 @@ use datafusion_common::{use async_trait::async_trait;use datafusion_datasource_json::file_format::JsonFormat;use datafusion_expr::SortExpr;+use futures::future::BoxFuture;/// Options that control the reading of CSV files.///@@ -602,26 +604,37 @@ pub trait ReadOptions<'a> {}/// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.- async fn _get_resolved_schema(+ fn _get_resolved_schema<'life0, 'async_trait>(&'a self,- config: &SessionConfig,+ config: &'life0 SessionConfig,state: SessionState,table_path: ListingTableUrl,schema: Option<&'a Schema>,- ) -> Result<SchemaRef>+ ) -> BoxFuture<'async_trait, Result<SchemaRef>>where'a: 'async_trait,+ 'life0: 'async_trait,+ Self: 'async_trait,{if let Some(s) = schema {- return Ok(Arc::new(s.to_owned()));+ return Box::pin(ready(Ok(Arc::new(s.to_owned()))));}- self.to_listing_options(config, state.default_table_options())- .infer_schema(&state, &table_path)- .await+ let listing_options =+ self.to_listing_options(config, state.default_table_options());+ infer_schema_boxed(listing_options, state, table_path)}}+/// Infers a schema from `table_path`, boxed for [`ReadOptions::_get_resolved_schema`].+fn infer_schema_boxed(+ listing_options: ListingOptions,+ state: SessionState,+ table_path: ListingTableUrl,+) -> BoxFuture<'static, Result<SchemaRef>> {+ Box::pin(async move { listing_options.infer_schema(&state, &table_path).await })+}+#[async_trait]impl ReadOptions<'_> for CsvReadOptions<'_> {fn to_listing_options(@@ -649,14 +662,18 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {.with_file_sort_order(self.file_sort_order.clone())}- async fn get_resolved_schema(- &self,- config: &SessionConfig,+ fn get_resolved_schema<'life0, 'life1, 'async_trait>(+ &'life0 self,+ config: &'life1 SessionConfig,state: SessionState,table_path: ListingTableUrl,- ) -> Result<SchemaRef> {+ ) -> BoxFuture<'async_trait, Result<SchemaRef>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {self._get_resolved_schema(config, state, table_path, self.schema)- .await}fn schema_source(&self) -> SchemaSource {@@ -696,14 +713,18 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {.with_file_sort_order(self.file_sort_order.clone())}- async fn get_resolved_schema(- &self,- config: &SessionConfig,+ fn get_resolved_schema<'life0, 'life1, 'async_trait>(+ &'life0 self,+ config: &'life1 SessionConfig,state: SessionState,table_path: ListingTableUrl,- ) -> Result<SchemaRef> {+ ) -> BoxFuture<'async_trait, Result<SchemaRef>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {self._get_resolved_schema(config, state, table_path, self.schema)- .await}fn schema_source(&self) -> SchemaSource {@@ -730,14 +751,18 @@ impl ReadOptions<'_> for JsonReadOptions<'_> {.with_file_sort_order(self.file_sort_order.clone())}- async fn get_resolved_schema(- &self,- config: &SessionConfig,+ fn get_resolved_schema<'life0, 'life1, 'async_trait>(+ &'life0 self,+ config: &'life1 SessionConfig,state: SessionState,table_path: ListingTableUrl,- ) -> Result<SchemaRef> {+ ) -> BoxFuture<'async_trait, Result<SchemaRef>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {self._get_resolved_schema(config, state, table_path, self.schema)- .await}fn schema_source(&self) -> SchemaSource {@@ -760,14 +785,18 @@ impl ReadOptions<'_> for AvroReadOptions<'_> {.with_table_partition_cols(self.table_partition_cols.clone())}- async fn get_resolved_schema(- &self,- config: &SessionConfig,+ fn get_resolved_schema<'life0, 'life1, 'async_trait>(+ &'life0 self,+ config: &'life1 SessionConfig,state: SessionState,table_path: ListingTableUrl,- ) -> Result<SchemaRef> {+ ) -> BoxFuture<'async_trait, Result<SchemaRef>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {self._get_resolved_schema(config, state, table_path, self.schema)- .await}fn schema_source(&self) -> SchemaSource {@@ -789,14 +818,18 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> {.with_table_partition_cols(self.table_partition_cols.clone())}- async fn get_resolved_schema(- &self,- config: &SessionConfig,+ fn get_resolved_schema<'life0, 'life1, 'async_trait>(+ &'life0 self,+ config: &'life1 SessionConfig,state: SessionState,table_path: ListingTableUrl,- ) -> Result<SchemaRef> {+ ) -> BoxFuture<'async_trait, Result<SchemaRef>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {self._get_resolved_schema(config, state, table_path, self.schema)- .await}fn schema_source(&self) -> SchemaSource {diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 56a5d57795..7b435822f2 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -21,6 +21,7 @@ use datafusion_catalog_listing::{ListingOptions, ListingTableConfig};use datafusion_common::{config_datafusion_err, internal_datafusion_err};use datafusion_session::Session;use futures::StreamExt;+use futures::future::BoxFuture;use std::collections::HashMap;/// Extension trait for [`ListingTableConfig`] that supports inferring schemas@@ -47,17 +48,50 @@ pub trait ListingTableConfigExt {#[async_trait]impl ListingTableConfigExt for ListingTableConfig {- async fn infer_options(+ fn infer_options<'life0, 'async_trait>(self,- state: &dyn Session,- ) -> datafusion_common::Result<ListingTableConfig> {- let store = if let Some(url) = self.table_paths.first() {+ state: &'life0 dyn Session,+ ) -> BoxFuture<'async_trait, datafusion_common::Result<ListingTableConfig>>+ where+ 'life0: 'async_trait,+ Self: 'async_trait,+ {+ infer_options_boxed(self, state)+ }++ fn infer<'life0, 'async_trait>(+ self,+ state: &'life0 dyn Session,+ ) -> BoxFuture<'async_trait, datafusion_common::Result<Self>>+ where+ 'life0: 'async_trait,+ Self: 'async_trait,+ {+ infer_boxed(self, state)+ }+}++/// Body of [`ListingTableConfigExt::infer`].+fn infer_boxed<'a>(+ config: ListingTableConfig,+ state: &'a dyn Session,+) -> BoxFuture<'a, datafusion_common::Result<ListingTableConfig>> {+ Box::pin(async move { config.infer_options(state).await?.infer_schema(state).await })+}++/// Body of [`ListingTableConfigExt::infer_options`].+fn infer_options_boxed<'a>(+ config: ListingTableConfig,+ state: &'a dyn Session,+) -> BoxFuture<'a, datafusion_common::Result<ListingTableConfig>> {+ Box::pin(async move {+ let store = if let Some(url) = config.table_paths.first() {state.runtime_env().object_store(url)?} else {- return Ok(self);+ return Ok(config);};- let file = self+ let file = config.table_paths.first().unwrap()@@ -95,12 +129,8 @@ impl ListingTableConfigExt for ListingTableConfig {let listing_options =ListingOptions::new(file_format).with_file_extension(listing_file_extension);- Ok(self.with_listing_options(listing_options))- }-- async fn infer(self, state: &dyn Session) -> datafusion_common::Result<Self> {- self.infer_options(state).await?.infer_schema(state).await- }+ Ok(config.with_listing_options(listing_options))+ })}#[cfg(test)]diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rsindex 68f6743189..116b8985b4 100644--- a/datafusion/core/src/datasource/listing_table_factory.rs+++ b/datafusion/core/src/datasource/listing_table_factory.rs@@ -36,6 +36,7 @@ use datafusion_expr::CreateExternalTable;use async_trait::async_trait;use datafusion_catalog::Session;+use futures::future::BoxFuture;/// A `TableProviderFactory` capable of creating new `ListingTable`s#[derive(Debug, Default)]@@ -50,7 +51,31 @@ impl ListingTableFactory {#[async_trait]impl TableProviderFactory for ListingTableFactory {- async fn create(+ fn create<'life0, 'life1, 'life2, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ cmd: &'life2 CreateExternalTable,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn TableProvider>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ Self: 'async_trait,+ {+ self.create_boxed(state, cmd)+ }+}++impl ListingTableFactory {+ fn create_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ cmd: &'a CreateExternalTable,+ ) -> BoxFuture<'a, Result<Arc<dyn TableProvider>>> {+ Box::pin(self.create_inner(state, cmd))+ }++ async fn create_inner(&self,state: &dyn Session,cmd: &CreateExternalTable,diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rsindex 9d4b67632a..84bb338835 100644--- a/datafusion/core/src/datasource/provider.rs+++ b/datafusion/core/src/datasource/provider.rs@@ -23,6 +23,7 @@ use async_trait::async_trait;use datafusion_catalog::Session;use datafusion_expr::CreateExternalTable;pub use datafusion_expr::{TableProviderFilterPushDown, TableType};+use futures::future::BoxFuture;use crate::catalog::{TableProvider, TableProviderFactory};use crate::datasource::listing_table_factory::ListingTableFactory;@@ -48,7 +49,31 @@ impl DefaultTableFactory {#[async_trait]impl TableProviderFactory for DefaultTableFactory {- async fn create(+ fn create<'life0, 'life1, 'life2, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ cmd: &'life2 CreateExternalTable,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn TableProvider>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ Self: 'async_trait,+ {+ self.create_boxed(state, cmd)+ }+}++impl DefaultTableFactory {+ fn create_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ cmd: &'a CreateExternalTable,+ ) -> BoxFuture<'a, Result<Arc<dyn TableProvider>>> {+ Box::pin(self.create_inner(state, cmd))+ }++ async fn create_inner(&self,state: &dyn Session,cmd: &CreateExternalTable,diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex bfd38faacf..99b16857bb 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -85,6 +85,7 @@ use datafusion_sql::{use async_trait::async_trait;use chrono::{DateTime, Utc};+use futures::future::BoxFuture;use itertools::Itertools;use log::{debug, info};use object_store::ObjectStore;@@ -291,11 +292,16 @@ impl Session for SessionState {SessionState::statistics_registry(self)}- async fn create_physical_plan(- &self,- logical_plan: &LogicalPlan,- ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {- self.create_physical_plan(logical_plan).await+ fn create_physical_plan<'life0, 'life1, 'async_trait>(+ &'life0 self,+ logical_plan: &'life1 LogicalPlan,+ ) -> BoxFuture<'async_trait, datafusion_common::Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ self.create_physical_plan_boxed(logical_plan)}fn create_physical_expr(@@ -351,6 +357,22 @@ impl Session for SessionState {}}+impl SessionState {+ fn create_physical_plan_boxed<'a>(+ &'a self,+ logical_plan: &'a LogicalPlan,+ ) -> BoxFuture<'a, datafusion_common::Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.create_physical_plan_inner(logical_plan))+ }++ async fn create_physical_plan_inner(+ &self,+ logical_plan: &LogicalPlan,+ ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {+ self.create_physical_plan(logical_plan).await+ }+}+impl SessionState {pub(crate) fn resolve_table_ref(&self,@@ -2336,7 +2358,31 @@ struct DefaultQueryPlanner {}#[async_trait]impl QueryPlanner for DefaultQueryPlanner {/// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution- async fn create_physical_plan(+ fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>(+ &'life0 self,+ logical_plan: &'life1 LogicalPlan,+ session_state: &'life2 dyn Session,+ ) -> BoxFuture<'async_trait, datafusion_common::Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ Self: 'async_trait,+ {+ self.create_physical_plan_boxed(logical_plan, session_state)+ }+}++impl DefaultQueryPlanner {+ fn create_physical_plan_boxed<'a>(+ &'a self,+ logical_plan: &'a LogicalPlan,+ session_state: &'a dyn Session,+ ) -> BoxFuture<'a, datafusion_common::Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.create_physical_plan_inner(logical_plan, session_state))+ }++ async fn create_physical_plan_inner(&self,logical_plan: &LogicalPlan,session_state: &dyn Session,diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 3c1e7b5078..6e68ed9ce6 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -112,6 +112,7 @@ use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Sessiouse async_trait::async_trait;use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper};+use futures::future::BoxFuture;use futures::{StreamExt, TryStreamExt};use indexmap::IndexSet;use itertools::{Itertools, multiunzip};@@ -153,22 +154,18 @@ pub struct DefaultPhysicalPlanner {#[async_trait]impl PhysicalPlanner for DefaultPhysicalPlanner {/// Create a physical plan from a logical plan- async fn create_physical_plan(- &self,- logical_plan: &LogicalPlan,- session_state: &dyn Session,- ) -> Result<Arc<dyn ExecutionPlan>> {- if let Some(plan) = self- .handle_explain_or_analyze(logical_plan, session_state)- .await?- {- return Ok(plan);- }- let plan = self- .create_initial_plan(logical_plan, session_state)- .await?;-- self.optimize_physical_plan(plan, session_state, |_, _| {})+ fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>(+ &'life0 self,+ logical_plan: &'life1 LogicalPlan,+ session_state: &'life2 dyn Session,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ Self: 'async_trait,+ {+ self.create_physical_plan_boxed(logical_plan, session_state)}/// Create a physical expression from a logical expression@@ -193,6 +190,34 @@ impl PhysicalPlanner for DefaultPhysicalPlanner {}}+impl DefaultPhysicalPlanner {+ fn create_physical_plan_boxed<'a>(+ &'a self,+ logical_plan: &'a LogicalPlan,+ session_state: &'a dyn Session,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.create_physical_plan_inner(logical_plan, session_state))+ }++ async fn create_physical_plan_inner(+ &self,+ logical_plan: &LogicalPlan,+ session_state: &dyn Session,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ if let Some(plan) = self+ .handle_explain_or_analyze(logical_plan, session_state)+ .await?+ {+ return Ok(plan);+ }+ let plan = self+ .create_initial_plan(logical_plan, session_state)+ .await?;++ self.optimize_physical_plan(plan, session_state, |_, _| {})+ }+}+#[derive(Debug)]struct ExecutionPlanChild {/// Index needed to order children of parent to ensure consistency with original@@ -329,7 +354,7 @@ impl DefaultPhysicalPlanner {&'a self,logical_plan: &'a LogicalPlan,session_state: &'a dyn Session,- ) -> futures::future::BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {Box::pin(async move {// When `enable_physical_uncorrelated_scalar_subquery` is disabled, the// `ScalarSubqueryToJoin` optimizer rule rewrites all uncorrelateddiff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rsindex d70c0d186d..036988c8fc 100644--- a/datafusion/core/src/test_util/mod.rs+++ b/datafusion/core/src/test_util/mod.rs@@ -26,6 +26,7 @@ use futures::Stream;use std::collections::HashMap;use std::fmt::Formatter;use std::fs::File;+use std::future::ready;use std::io::Write;use std::path::Path;use std::sync::Arc;@@ -53,6 +54,7 @@ use datafusion_expr::{use std::pin::Pin;use async_trait::async_trait;+use futures::future::BoxFuture;use tempfile::TempDir;// backwards compatibility@@ -182,9 +184,33 @@ pub struct TestTableFactory {}#[async_trait]impl TableProviderFactory for TestTableFactory {- async fn create(+ fn create<'life0, 'life1, 'life2, 'async_trait>(+ &'life0 self,+ session: &'life1 dyn Session,+ cmd: &'life2 CreateExternalTable,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn TableProvider>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ Self: 'async_trait,+ {+ self.create_boxed(session, cmd)+ }+}++impl TestTableFactory {+ fn create_boxed<'a>(+ &'a self,+ session: &'a dyn Session,+ cmd: &'a CreateExternalTable,+ ) -> BoxFuture<'a, Result<Arc<dyn TableProvider>>> {+ Box::pin(ready(self.create_inner(session, cmd)))+ }++ fn create_inner(&self,- _: &dyn Session,+ _session: &dyn Session,cmd: &CreateExternalTable,) -> Result<Arc<dyn TableProvider>> {let Some(location) = cmd.locations.first() else {