Skip to content

Apache DataFusion SQL Query Engine

perf(catalog-listing): cut `datafusion-catalog-listing` compile time ~4.4x (#24330)

Daniël Heres519687ca22ff
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs
index b3328cc063..46fab1f9cb 100644
--- a/datafusion/catalog-listing/src/table.rs
+++ b/datafusion/catalog-listing/src/table.rs
@@ -53,6 +53,7 @@ use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::empty::EmptyExec;
+use futures::future::BoxFuture;
use futures::{Stream, StreamExt, TryStreamExt, future, stream};
use object_store::ObjectStore;
use std::collections::{HashMap, HashSet};
@@ -489,21 +490,94 @@ impl TableProvider for ListingTable {
TableType::Base
}
- async fn scan(
- &self,
- state: &dyn Session,
- projection: Option<&Vec<usize>>,
- filters: &[Expr],
+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ projection: Option<&'life2 Vec<usize>>,
+ filters: &'life3 [Expr],
limit: Option<usize>,
- ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
- let options = ScanArgs::default()
- .with_projection(projection.map(|p| p.as_slice()))
- .with_filters(Some(filters))
- .with_limit(limit);
- Ok(self.scan_with_args(state, options).await?.into_inner())
+ ) -> BoxFuture<'async_trait, datafusion_common::Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ 'life2: 'async_trait,
+ 'life3: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.scan_boxed(state, projection, filters, limit)
+ }
+
+ fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ args: ScanArgs<'a>,
+ ) -> BoxFuture<'async_trait, datafusion_common::Result<ScanResult>>
+ where
+ 'a: 'async_trait,
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.scan_with_args_boxed(state, args)
+ }
+
+ fn supports_filters_pushdown(
+ &self,
+ filters: &[&Expr],
+ ) -> datafusion_common::Result<Vec<TableProviderFilterPushDown>> {
+ let partition_column_names = self
+ .options
+ .table_partition_cols
+ .iter()
+ .map(|col| col.0.as_str())
+ .collect::<Vec<_>>();
+ filters
+ .iter()
+ .map(|filter| {
+ if can_be_evaluated_for_partition_pruning(&partition_column_names, filter)
+ {
+ // if filter can be handled by partition pruning, it is exact
+ return Ok(TableProviderFilterPushDown::Exact);
+ }
+
+ Ok(TableProviderFilterPushDown::Inexact)
+ })
+ .collect()
+ }
+
+ fn get_table_definition(&self) -> Option<&str> {
+ self.definition.as_deref()
+ }
+
+ fn insert_into<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ input: Arc<dyn ExecutionPlan>,
+ insert_op: InsertOp,
+ ) -> BoxFuture<'async_trait, datafusion_common::Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.insert_into_boxed(state, input, insert_op)
+ }
+
+ fn get_column_default(&self, column: &str) -> Option<&Expr> {
+ self.column_defaults.get(column)
+ }
+}
+
+impl ListingTable {
+ fn scan_with_args_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ args: ScanArgs<'a>,
+ ) -> BoxFuture<'a, datafusion_common::Result<ScanResult>> {
+ Box::pin(self.scan_with_args_inner(state, args))
}
- async fn scan_with_args<'a>(
+ async fn scan_with_args_inner<'a>(
&self,
state: &dyn Session,
args: ScanArgs<'a>,
@@ -670,35 +744,40 @@ impl TableProvider for ListingTable {
Ok(ScanResult::new(plan))
}
- fn supports_filters_pushdown(
- &self,
- filters: &[&Expr],
- ) -> datafusion_common::Result<Vec<TableProviderFilterPushDown>> {
- let partition_column_names = self
- .options
- .table_partition_cols
- .iter()
- .map(|col| col.0.as_str())
- .collect::<Vec<_>>();
- filters
- .iter()
- .map(|filter| {
- if can_be_evaluated_for_partition_pruning(&partition_column_names, filter)
- {
- // if filter can be handled by partition pruning, it is exact
- return Ok(TableProviderFilterPushDown::Exact);
- }
+ fn scan_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ projection: Option<&'a Vec<usize>>,
+ filters: &'a [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'a, datafusion_common::Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(self.scan_inner(state, projection, filters, limit))
+ }
- Ok(TableProviderFilterPushDown::Inexact)
- })
- .collect()
+ async fn scan_inner(
+ &self,
+ state: &dyn Session,
+ projection: Option<&Vec<usize>>,
+ filters: &[Expr],
+ limit: Option<usize>,
+ ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
+ let options = ScanArgs::default()
+ .with_projection(projection.map(|p| p.as_slice()))
+ .with_filters(Some(filters))
+ .with_limit(limit);
+ Ok(self.scan_with_args(state, options).await?.into_inner())
}
- fn get_table_definition(&self) -> Option<&str> {
- self.definition.as_deref()
+ fn insert_into_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ input: Arc<dyn ExecutionPlan>,
+ insert_op: InsertOp,
+ ) -> BoxFuture<'a, datafusion_common::Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(self.insert_into_inner(state, input, insert_op))
}
- async fn insert_into(
+ async fn insert_into_inner(
&self,
state: &dyn Session,
input: Arc<dyn ExecutionPlan>,
@@ -766,10 +845,6 @@ impl TableProvider for ListingTable {
.create_writer_physical_plan(input, state, config, order_requirements)
.await
}
-
- fn get_column_default(&self, column: &str) -> Option<&Expr> {
- self.column_defaults.get(column)
- }
}
impl ListingTable {