Skip to content

Apache DataFusion SQL Query Engine

perf(catalog): cut `datafusion-catalog` compile time ~6x (#24325)

Daniël Heres1a770de07e71
diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rs
index 5ec688526c..37f66b8c56 100644
--- a/datafusion/catalog/src/cte_worktable.rs
+++ b/datafusion/catalog/src/cte_worktable.rs
@@ -18,6 +18,7 @@
//! CteWorkTable implementation used for recursive queries
use std::borrow::Cow;
+use std::future::ready;
use std::sync::Arc;
use arrow::datatypes::SchemaRef;
@@ -26,6 +27,7 @@ use datafusion_common::error::Result;
use datafusion_expr::{Expr, LogicalPlan, TableProviderFilterPushDown, TableType};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::work_table::WorkTableExec;
+use futures::future::BoxFuture;
use crate::{ScanArgs, ScanResult, Session, TableProvider};
@@ -78,30 +80,35 @@ impl TableProvider for CteWorkTable {
TableType::Temporary
}
- async fn scan(
- &self,
- state: &dyn Session,
- projection: Option<&Vec<usize>>,
- filters: &[Expr],
+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ projection: Option<&'life2 Vec<usize>>,
+ filters: &'life3 [Expr],
limit: Option<usize>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- let options = ScanArgs::default()
- .with_projection(projection.map(|p| p.as_slice()))
- .with_filters(Some(filters))
- .with_limit(limit);
- Ok(self.scan_with_args(state, options).await?.into_inner())
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ 'life2: 'async_trait,
+ 'life3: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.scan_boxed(state, projection, filters, limit)
}
- async fn scan_with_args<'a>(
- &self,
- _state: &dyn Session,
+ fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
args: ScanArgs<'a>,
- ) -> Result<ScanResult> {
- Ok(ScanResult::new(Arc::new(WorkTableExec::new(
- self.name.clone(),
- Arc::clone(&self.table_schema),
- args.projection().map(|p| p.to_vec()),
- )?)))
+ ) -> BoxFuture<'async_trait, Result<ScanResult>>
+ where
+ 'a: 'async_trait,
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ Box::pin(ready(self.scan_with_args_inner(state, &args)))
}
fn supports_filters_pushdown(
@@ -115,3 +122,41 @@ impl TableProvider for CteWorkTable {
])
}
}
+
+impl CteWorkTable {
+ fn scan_with_args_inner<'a>(
+ &self,
+ _state: &dyn Session,
+ args: &ScanArgs<'a>,
+ ) -> Result<ScanResult> {
+ Ok(ScanResult::new(Arc::new(WorkTableExec::new(
+ self.name.clone(),
+ Arc::clone(&self.table_schema),
+ args.projection().map(|p| p.to_vec()),
+ )?)))
+ }
+
+ fn scan_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ projection: Option<&'a Vec<usize>>,
+ filters: &'a [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(self.scan_inner(state, projection, filters, limit))
+ }
+
+ async fn scan_inner(
+ &self,
+ state: &dyn Session,
+ projection: Option<&Vec<usize>>,
+ filters: &[Expr],
+ limit: Option<usize>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let options = ScanArgs::default()
+ .with_projection(projection.map(|p| p.as_slice()))
+ .with_filters(Some(filters))
+ .with_limit(limit);
+ Ok(self.scan_with_args(state, options).await?.into_inner())
+ }
+}
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs
index 4cf96cb364..79c6e44933 100644
--- a/datafusion/catalog/src/memory/table.rs
+++ b/datafusion/catalog/src/memory/table.rs
@@ -19,6 +19,7 @@
use std::collections::HashMap;
use std::fmt::Debug;
+use std::future::ready;
use std::sync::Arc;
use crate::TableProvider;
@@ -51,6 +52,7 @@ use datafusion_physical_plan::{
use datafusion_session::Session;
use async_trait::async_trait;
+use futures::future::BoxFuture;
use log::debug;
use parking_lot::Mutex;
use tokio::sync::RwLock;
@@ -184,7 +186,95 @@ impl TableProvider for MemTable {
TableType::Base
}
- async fn scan(
+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ projection: Option<&'life2 Vec<usize>>,
+ filters: &'life3 [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ 'life2: 'async_trait,
+ 'life3: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.scan_boxed(state, projection, filters, limit)
+ }
+
+ /// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`].
+ ///
+ /// The [`ExecutionPlan`] must have the same schema as this [`MemTable`].
+ ///
+ /// # Arguments
+ ///
+ /// * `state` - The [`SessionState`] containing the context for executing the plan.
+ /// * `input` - The [`ExecutionPlan`] to execute and insert.
+ ///
+ /// # Returns
+ ///
+ /// * A plan that returns the number of rows written.
+ ///
+ /// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
+ fn insert_into<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ input: Arc<dyn ExecutionPlan>,
+ insert_op: InsertOp,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.insert_into_boxed(state, input, insert_op)
+ }
+
+ fn get_column_default(&self, column: &str) -> Option<&Expr> {
+ self.column_defaults.get(column)
+ }
+
+ fn delete_from<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ filters: Vec<Expr>,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.delete_from_boxed(state, filters)
+ }
+
+ fn update<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ assignments: Vec<(String, Expr)>,
+ filters: Vec<Expr>,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.update_boxed(state, assignments, filters)
+ }
+}
+
+impl MemTable {
+ fn scan_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ projection: Option<&'a Vec<usize>>,
+ filters: &'a [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(self.scan_inner(state, projection, filters, limit))
+ }
+
+ async fn scan_inner(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
@@ -225,21 +315,16 @@ impl TableProvider for MemTable {
Ok(DataSourceExec::from_data_source(source))
}
- /// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`].
- ///
- /// The [`ExecutionPlan`] must have the same schema as this [`MemTable`].
- ///
- /// # Arguments
- ///
- /// * `state` - The [`SessionState`] containing the context for executing the plan.
- /// * `input` - The [`ExecutionPlan`] to execute and insert.
- ///
- /// # Returns
- ///
- /// * A plan that returns the number of rows written.
- ///
- /// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
- async fn insert_into(
+ fn insert_into_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ input: Arc<dyn ExecutionPlan>,
+ insert_op: InsertOp,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(ready(self.insert_into_inner(state, input, insert_op)))
+ }
+
+ fn insert_into_inner(
&self,
_state: &dyn Session,
input: Arc<dyn ExecutionPlan>,
@@ -260,11 +345,15 @@ impl TableProvider for MemTable {
Ok(Arc::new(DataSinkExec::new(input, Arc::new(sink), None)))
}
- fn get_column_default(&self, column: &str) -> Option<&Expr> {
- self.column_defaults.get(column)
+ fn delete_from_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ filters: Vec<Expr>,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(self.delete_from_inner(state, filters))
}
- async fn delete_from(
+ async fn delete_from_inner(
&self,
state: &dyn Session,
filters: Vec<Expr>,
@@ -328,7 +417,16 @@ impl TableProvider for MemTable {
Ok(Arc::new(DmlResultExec::new(total_deleted)))
}
- async fn update(
+ fn update_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ assignments: Vec<(String, Expr)>,
+ filters: Vec<Expr>,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(self.update_inner(state, assignments, filters))
+ }
+
+ async fn update_inner(
&self,
state: &dyn Session,
assignments: Vec<(String, Expr)>,
diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rs
index c8060456dd..f01874612e 100644
--- a/datafusion/catalog/src/stream.rs
+++ b/datafusion/catalog/src/stream.rs
@@ -19,6 +19,7 @@
use std::fmt::Formatter;
use std::fs::{File, OpenOptions};
+use std::future::ready;
use std::io::BufReader;
use std::path::PathBuf;
use std::str::FromStr;
@@ -40,6 +41,7 @@ use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};
use async_trait::async_trait;
use futures::StreamExt;
+use futures::future::BoxFuture;
/// A [`TableProviderFactory`] for [`StreamTable`]
#[derive(Debug, Default)]
@@ -47,7 +49,31 @@ pub struct StreamTableFactory {}
#[async_trait]
impl TableProviderFactory for StreamTableFactory {
- async fn create(
+ fn create<'life0, 'life1, 'life2, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ cmd: &'life2 CreateExternalTable,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn TableProvider>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ 'life2: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.create_boxed(state, cmd)
+ }
+}
+
+impl StreamTableFactory {
+ fn create_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ cmd: &'a CreateExternalTable,
+ ) -> BoxFuture<'a, Result<Arc<dyn TableProvider>>> {
+ Box::pin(ready(self.create_inner(state, cmd)))
+ }
+
+ fn create_inner(
&self,
state: &dyn Session,
cmd: &CreateExternalTable,
@@ -322,7 +348,50 @@ impl TableProvider for StreamTable {
TableType::Base
}
- async fn scan(
+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ projection: Option<&'life2 Vec<usize>>,
+ filters: &'life3 [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ 'life2: 'async_trait,
+ 'life3: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.scan_boxed(state, projection, filters, limit)
+ }
+
+ fn insert_into<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ input: Arc<dyn ExecutionPlan>,
+ insert_op: InsertOp,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.insert_into_boxed(state, input, insert_op)
+ }
+}
+
+impl StreamTable {
+ fn scan_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ projection: Option<&'a Vec<usize>>,
+ filters: &'a [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(ready(self.scan_inner(state, projection, filters, limit)))
+ }
+
+ fn scan_inner(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
@@ -351,15 +420,23 @@ impl TableProvider for StreamTable {
)?))
}
- async fn insert_into(
+ fn insert_into_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ input: Arc<dyn ExecutionPlan>,
+ insert_op: InsertOp,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(ready(self.insert_into_inner(state, input, insert_op)))
+ }
+
+ fn insert_into_inner(
&self,
- _state: &dyn Session,
+ state: &dyn Session,
input: Arc<dyn ExecutionPlan>,
_insert_op: InsertOp,
) -> Result<Arc<dyn ExecutionPlan>> {
let schema = self.0.source.schema();
- let orders =
- create_lex_ordering(schema, &self.0.order, _state.execution_props())?;
+ let orders = create_lex_ordering(schema, &self.0.order, state.execution_props())?;
// It is sufficient to pass only one of the equivalent orderings:
let ordering = orders.into_iter().next().map(Into::into);
@@ -412,7 +489,30 @@ impl DataSink for StreamWrite {
self.0.source.schema()
}
- async fn write_all(
+ fn write_all<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ data: SendableRecordBatchStream,
+ context: &'life1 Arc<TaskContext>,
+ ) -> BoxFuture<'async_trait, Result<u64>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.write_all_boxed(data, context)
+ }
+}
+
+impl StreamWrite {
+ fn write_all_boxed<'a>(
+ &'a self,
+ data: SendableRecordBatchStream,
+ context: &'a Arc<TaskContext>,
+ ) -> BoxFuture<'a, Result<u64>> {
+ Box::pin(self.write_all_inner(data, context))
+ }
+
+ async fn write_all_inner(
&self,
mut data: SendableRecordBatchStream,
_context: &Arc<TaskContext>,
diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rs
index 50f05355aa..bb70270ce9 100644
--- a/datafusion/catalog/src/streaming.rs
+++ b/datafusion/catalog/src/streaming.rs
@@ -17,6 +17,7 @@
//! A simplified [`TableProvider`] for streaming partitioned datasets
+use std::future::ready;
use std::sync::Arc;
use arrow::datatypes::SchemaRef;
@@ -31,6 +32,7 @@ use datafusion_physical_expr::{
};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};
+use futures::future::BoxFuture;
use log::debug;
use crate::{Session, TableProvider};
@@ -121,7 +123,36 @@ impl TableProvider for StreamingTable {
TableType::View
}
- async fn scan(
+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ projection: Option<&'life2 Vec<usize>>,
+ filters: &'life3 [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ 'life2: 'async_trait,
+ 'life3: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.scan_boxed(state, projection, filters, limit)
+ }
+}
+
+impl StreamingTable {
+ fn scan_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ projection: Option<&'a Vec<usize>>,
+ filters: &'a [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(ready(self.scan_inner(state, projection, filters, limit)))
+ }
+
+ fn scan_inner(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rs
index 723634c34c..ac0f74c181 100644
--- a/datafusion/catalog/src/view.rs
+++ b/datafusion/catalog/src/view.rs
@@ -30,6 +30,7 @@ use datafusion_expr::TableType;
use datafusion_expr::{Expr, LogicalPlan};
use datafusion_expr::{LogicalPlanBuilder, TableProviderFilterPushDown};
use datafusion_physical_plan::ExecutionPlan;
+use futures::future::BoxFuture;
/// An implementation of `TableProvider` that uses another logical plan.
#[derive(Debug)]
@@ -95,7 +96,36 @@ impl TableProvider for ViewTable {
Ok(vec![TableProviderFilterPushDown::Exact; filters.len()])
}
- async fn scan(
+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
+ &'life0 self,
+ state: &'life1 dyn Session,
+ projection: Option<&'life2 Vec<usize>>,
+ filters: &'life3 [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ 'life2: 'async_trait,
+ 'life3: 'async_trait,
+ Self: 'async_trait,
+ {
+ self.scan_boxed(state, projection, filters, limit)
+ }
+}
+
+impl ViewTable {
+ fn scan_boxed<'a>(
+ &'a self,
+ state: &'a dyn Session,
+ projection: Option<&'a Vec<usize>>,
+ filters: &'a [Expr],
+ limit: Option<usize>,
+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
+ Box::pin(self.scan_inner(state, projection, filters, limit))
+ }
+
+ async fn scan_inner(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,