Apache DataFusion SQL Query Engine
perf(catalog): cut `datafusion-catalog` compile time ~6x (#24325)
Daniël Heres1a770de07e71
diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rsindex 5ec688526c..37f66b8c56 100644--- a/datafusion/catalog/src/cte_worktable.rs+++ b/datafusion/catalog/src/cte_worktable.rs@@ -18,6 +18,7 @@//! CteWorkTable implementation used for recursive queriesuse std::borrow::Cow;+use std::future::ready;use std::sync::Arc;use arrow::datatypes::SchemaRef;@@ -26,6 +27,7 @@ use datafusion_common::error::Result;use datafusion_expr::{Expr, LogicalPlan, TableProviderFilterPushDown, TableType};use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::work_table::WorkTableExec;+use futures::future::BoxFuture;use crate::{ScanArgs, ScanResult, Session, TableProvider};@@ -78,30 +80,35 @@ impl TableProvider for CteWorkTable {TableType::Temporary}- async fn scan(- &self,- state: &dyn Session,- projection: Option<&Vec<usize>>,- filters: &[Expr],+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ projection: Option<&'life2 Vec<usize>>,+ filters: &'life3 [Expr],limit: Option<usize>,- ) -> Result<Arc<dyn ExecutionPlan>> {- let options = ScanArgs::default()- .with_projection(projection.map(|p| p.as_slice()))- .with_filters(Some(filters))- .with_limit(limit);- Ok(self.scan_with_args(state, options).await?.into_inner())+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ 'life3: 'async_trait,+ Self: 'async_trait,+ {+ self.scan_boxed(state, projection, filters, limit)}- async fn scan_with_args<'a>(- &self,- _state: &dyn Session,+ fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,args: ScanArgs<'a>,- ) -> Result<ScanResult> {- Ok(ScanResult::new(Arc::new(WorkTableExec::new(- self.name.clone(),- Arc::clone(&self.table_schema),- args.projection().map(|p| p.to_vec()),- )?)))+ ) -> BoxFuture<'async_trait, Result<ScanResult>>+ where+ 'a: 'async_trait,+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ Box::pin(ready(self.scan_with_args_inner(state, &args)))}fn supports_filters_pushdown(@@ -115,3 +122,41 @@ impl TableProvider for CteWorkTable {])}}++impl CteWorkTable {+ fn scan_with_args_inner<'a>(+ &self,+ _state: &dyn Session,+ args: &ScanArgs<'a>,+ ) -> Result<ScanResult> {+ Ok(ScanResult::new(Arc::new(WorkTableExec::new(+ self.name.clone(),+ Arc::clone(&self.table_schema),+ args.projection().map(|p| p.to_vec()),+ )?)))+ }++ fn scan_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ projection: Option<&'a Vec<usize>>,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.scan_inner(state, projection, filters, limit))+ }++ async fn scan_inner(+ &self,+ state: &dyn Session,+ projection: Option<&Vec<usize>>,+ filters: &[Expr],+ limit: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let options = ScanArgs::default()+ .with_projection(projection.map(|p| p.as_slice()))+ .with_filters(Some(filters))+ .with_limit(limit);+ Ok(self.scan_with_args(state, options).await?.into_inner())+ }+}diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 4cf96cb364..79c6e44933 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -19,6 +19,7 @@use std::collections::HashMap;use std::fmt::Debug;+use std::future::ready;use std::sync::Arc;use crate::TableProvider;@@ -51,6 +52,7 @@ use datafusion_physical_plan::{use datafusion_session::Session;use async_trait::async_trait;+use futures::future::BoxFuture;use log::debug;use parking_lot::Mutex;use tokio::sync::RwLock;@@ -184,7 +186,95 @@ impl TableProvider for MemTable {TableType::Base}- async fn scan(+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ projection: Option<&'life2 Vec<usize>>,+ filters: &'life3 [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ 'life3: 'async_trait,+ Self: 'async_trait,+ {+ self.scan_boxed(state, projection, filters, limit)+ }++ /// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`].+ ///+ /// The [`ExecutionPlan`] must have the same schema as this [`MemTable`].+ ///+ /// # Arguments+ ///+ /// * `state` - The [`SessionState`] containing the context for executing the plan.+ /// * `input` - The [`ExecutionPlan`] to execute and insert.+ ///+ /// # Returns+ ///+ /// * A plan that returns the number of rows written.+ ///+ /// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html+ fn insert_into<'life0, 'life1, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ input: Arc<dyn ExecutionPlan>,+ insert_op: InsertOp,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ self.insert_into_boxed(state, input, insert_op)+ }++ fn get_column_default(&self, column: &str) -> Option<&Expr> {+ self.column_defaults.get(column)+ }++ fn delete_from<'life0, 'life1, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ filters: Vec<Expr>,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ self.delete_from_boxed(state, filters)+ }++ fn update<'life0, 'life1, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ assignments: Vec<(String, Expr)>,+ filters: Vec<Expr>,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ self.update_boxed(state, assignments, filters)+ }+}++impl MemTable {+ fn scan_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ projection: Option<&'a Vec<usize>>,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.scan_inner(state, projection, filters, limit))+ }++ async fn scan_inner(&self,state: &dyn Session,projection: Option<&Vec<usize>>,@@ -225,21 +315,16 @@ impl TableProvider for MemTable {Ok(DataSourceExec::from_data_source(source))}- /// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`].- ///- /// The [`ExecutionPlan`] must have the same schema as this [`MemTable`].- ///- /// # Arguments- ///- /// * `state` - The [`SessionState`] containing the context for executing the plan.- /// * `input` - The [`ExecutionPlan`] to execute and insert.- ///- /// # Returns- ///- /// * A plan that returns the number of rows written.- ///- /// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html- async fn insert_into(+ fn insert_into_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ input: Arc<dyn ExecutionPlan>,+ insert_op: InsertOp,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(ready(self.insert_into_inner(state, input, insert_op)))+ }++ fn insert_into_inner(&self,_state: &dyn Session,input: Arc<dyn ExecutionPlan>,@@ -260,11 +345,15 @@ impl TableProvider for MemTable {Ok(Arc::new(DataSinkExec::new(input, Arc::new(sink), None)))}- fn get_column_default(&self, column: &str) -> Option<&Expr> {- self.column_defaults.get(column)+ fn delete_from_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ filters: Vec<Expr>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.delete_from_inner(state, filters))}- async fn delete_from(+ async fn delete_from_inner(&self,state: &dyn Session,filters: Vec<Expr>,@@ -328,7 +417,16 @@ impl TableProvider for MemTable {Ok(Arc::new(DmlResultExec::new(total_deleted)))}- async fn update(+ fn update_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ assignments: Vec<(String, Expr)>,+ filters: Vec<Expr>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.update_inner(state, assignments, filters))+ }++ async fn update_inner(&self,state: &dyn Session,assignments: Vec<(String, Expr)>,diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rsindex c8060456dd..f01874612e 100644--- a/datafusion/catalog/src/stream.rs+++ b/datafusion/catalog/src/stream.rs@@ -19,6 +19,7 @@use std::fmt::Formatter;use std::fs::{File, OpenOptions};+use std::future::ready;use std::io::BufReader;use std::path::PathBuf;use std::str::FromStr;@@ -40,6 +41,7 @@ use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};use async_trait::async_trait;use futures::StreamExt;+use futures::future::BoxFuture;/// A [`TableProviderFactory`] for [`StreamTable`]#[derive(Debug, Default)]@@ -47,7 +49,31 @@ pub struct StreamTableFactory {}#[async_trait]impl TableProviderFactory for StreamTableFactory {- async fn create(+ fn create<'life0, 'life1, 'life2, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ cmd: &'life2 CreateExternalTable,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn TableProvider>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ Self: 'async_trait,+ {+ self.create_boxed(state, cmd)+ }+}++impl StreamTableFactory {+ fn create_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ cmd: &'a CreateExternalTable,+ ) -> BoxFuture<'a, Result<Arc<dyn TableProvider>>> {+ Box::pin(ready(self.create_inner(state, cmd)))+ }++ fn create_inner(&self,state: &dyn Session,cmd: &CreateExternalTable,@@ -322,7 +348,50 @@ impl TableProvider for StreamTable {TableType::Base}- async fn scan(+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ projection: Option<&'life2 Vec<usize>>,+ filters: &'life3 [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ 'life3: 'async_trait,+ Self: 'async_trait,+ {+ self.scan_boxed(state, projection, filters, limit)+ }++ fn insert_into<'life0, 'life1, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ input: Arc<dyn ExecutionPlan>,+ insert_op: InsertOp,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ self.insert_into_boxed(state, input, insert_op)+ }+}++impl StreamTable {+ fn scan_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ projection: Option<&'a Vec<usize>>,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(ready(self.scan_inner(state, projection, filters, limit)))+ }++ fn scan_inner(&self,state: &dyn Session,projection: Option<&Vec<usize>>,@@ -351,15 +420,23 @@ impl TableProvider for StreamTable {)?))}- async fn insert_into(+ fn insert_into_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ input: Arc<dyn ExecutionPlan>,+ insert_op: InsertOp,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(ready(self.insert_into_inner(state, input, insert_op)))+ }++ fn insert_into_inner(&self,- _state: &dyn Session,+ state: &dyn Session,input: Arc<dyn ExecutionPlan>,_insert_op: InsertOp,) -> Result<Arc<dyn ExecutionPlan>> {let schema = self.0.source.schema();- let orders =- create_lex_ordering(schema, &self.0.order, _state.execution_props())?;+ let orders = create_lex_ordering(schema, &self.0.order, state.execution_props())?;// It is sufficient to pass only one of the equivalent orderings:let ordering = orders.into_iter().next().map(Into::into);@@ -412,7 +489,30 @@ impl DataSink for StreamWrite {self.0.source.schema()}- async fn write_all(+ fn write_all<'life0, 'life1, 'async_trait>(+ &'life0 self,+ data: SendableRecordBatchStream,+ context: &'life1 Arc<TaskContext>,+ ) -> BoxFuture<'async_trait, Result<u64>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ self.write_all_boxed(data, context)+ }+}++impl StreamWrite {+ fn write_all_boxed<'a>(+ &'a self,+ data: SendableRecordBatchStream,+ context: &'a Arc<TaskContext>,+ ) -> BoxFuture<'a, Result<u64>> {+ Box::pin(self.write_all_inner(data, context))+ }++ async fn write_all_inner(&self,mut data: SendableRecordBatchStream,_context: &Arc<TaskContext>,diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rsindex 50f05355aa..bb70270ce9 100644--- a/datafusion/catalog/src/streaming.rs+++ b/datafusion/catalog/src/streaming.rs@@ -17,6 +17,7 @@//! A simplified [`TableProvider`] for streaming partitioned datasets+use std::future::ready;use std::sync::Arc;use arrow::datatypes::SchemaRef;@@ -31,6 +32,7 @@ use datafusion_physical_expr::{};use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};+use futures::future::BoxFuture;use log::debug;use crate::{Session, TableProvider};@@ -121,7 +123,36 @@ impl TableProvider for StreamingTable {TableType::View}- async fn scan(+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ projection: Option<&'life2 Vec<usize>>,+ filters: &'life3 [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ 'life3: 'async_trait,+ Self: 'async_trait,+ {+ self.scan_boxed(state, projection, filters, limit)+ }+}++impl StreamingTable {+ fn scan_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ projection: Option<&'a Vec<usize>>,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(ready(self.scan_inner(state, projection, filters, limit)))+ }++ fn scan_inner(&self,state: &dyn Session,projection: Option<&Vec<usize>>,diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rsindex 723634c34c..ac0f74c181 100644--- a/datafusion/catalog/src/view.rs+++ b/datafusion/catalog/src/view.rs@@ -30,6 +30,7 @@ use datafusion_expr::TableType;use datafusion_expr::{Expr, LogicalPlan};use datafusion_expr::{LogicalPlanBuilder, TableProviderFilterPushDown};use datafusion_physical_plan::ExecutionPlan;+use futures::future::BoxFuture;/// An implementation of `TableProvider` that uses another logical plan.#[derive(Debug)]@@ -95,7 +96,36 @@ impl TableProvider for ViewTable {Ok(vec![TableProviderFilterPushDown::Exact; filters.len()])}- async fn scan(+ fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,+ projection: Option<&'life2 Vec<usize>>,+ filters: &'life3 [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>+ where+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ 'life2: 'async_trait,+ 'life3: 'async_trait,+ Self: 'async_trait,+ {+ self.scan_boxed(state, projection, filters, limit)+ }+}++impl ViewTable {+ fn scan_boxed<'a>(+ &'a self,+ state: &'a dyn Session,+ projection: Option<&'a Vec<usize>>,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {+ Box::pin(self.scan_inner(state, projection, filters, limit))+ }++ async fn scan_inner(&self,state: &dyn Session,projection: Option<&Vec<usize>>,