Apache DataFusion SQL Query Engine
Docs: document manual catalog impls and how compilation got faster (#24362)
Andrew Lamb1b67f2e9d6ad
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 46fab1f9cb..498640ba0f 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -490,6 +490,8 @@ impl TableProvider for ListingTable {TableType::Base}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -507,6 +509,8 @@ impl TableProvider for ListingTable {self.scan_boxed(state, projection, filters, limit)}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -549,6 +553,8 @@ impl TableProvider for ListingTable {self.definition.as_deref()}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn insert_into<'life0, 'life1, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rsindex 37f66b8c56..6690fad24c 100644--- a/datafusion/catalog/src/cte_worktable.rs+++ b/datafusion/catalog/src/cte_worktable.rs@@ -80,6 +80,8 @@ impl TableProvider for CteWorkTable {TableType::Temporary}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -97,6 +99,8 @@ impl TableProvider for CteWorkTable {self.scan_boxed(state, projection, filters, limit)}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 79c6e44933..8bb45e945d 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -186,6 +186,8 @@ impl TableProvider for MemTable {TableType::Base}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -217,6 +219,8 @@ impl TableProvider for MemTable {/// * A plan that returns the number of rows written.////// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn insert_into<'life0, 'life1, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -235,6 +239,8 @@ impl TableProvider for MemTable {self.column_defaults.get(column)}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn delete_from<'life0, 'life1, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -248,6 +254,8 @@ impl TableProvider for MemTable {self.delete_from_boxed(state, filters)}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn update<'life0, 'life1, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rsindex f01874612e..86bd4e81dd 100644--- a/datafusion/catalog/src/stream.rs+++ b/datafusion/catalog/src/stream.rs@@ -49,6 +49,8 @@ pub struct StreamTableFactory {}#[async_trait]impl TableProviderFactory for StreamTableFactory {+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -348,6 +350,8 @@ impl TableProvider for StreamTable {TableType::Base}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -365,6 +369,8 @@ impl TableProvider for StreamTable {self.scan_boxed(state, projection, filters, limit)}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn insert_into<'life0, 'life1, 'async_trait>(&'life0 self,state: &'life1 dyn Session,@@ -489,6 +495,8 @@ impl DataSink for StreamWrite {self.0.source.schema()}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn write_all<'life0, 'life1, 'async_trait>(&'life0 self,data: SendableRecordBatchStream,diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rsindex bb70270ce9..7b466d8a59 100644--- a/datafusion/catalog/src/streaming.rs+++ b/datafusion/catalog/src/streaming.rs@@ -123,6 +123,8 @@ impl TableProvider for StreamingTable {TableType::View}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rsindex ac0f74c181..32af208984 100644--- a/datafusion/catalog/src/view.rs+++ b/datafusion/catalog/src/view.rs@@ -96,6 +96,8 @@ impl TableProvider for ViewTable {Ok(vec![TableProviderFilterPushDown::Exact; filters.len()])}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 8be23f927e..5ca40c854b 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -2717,6 +2717,8 @@ impl TableProvider for DataFrameTableProvider {self.table_type}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rsindex f670c9cc44..0212222cdd 100644--- a/datafusion/core/src/datasource/dynamic_file.rs+++ b/datafusion/core/src/datasource/dynamic_file.rs@@ -54,6 +54,8 @@ impl DynamicListTableFactory {#[async_trait]impl UrlTableFactory for DynamicListTableFactory {+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn try_new<'life0, 'life1, 'async_trait>(&'life0 self,url: &'life1 str,diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rsindex 4babf4a421..f2903405c8 100644--- a/datafusion/core/src/datasource/file_format/options.rs+++ b/datafusion/core/src/datasource/file_format/options.rs@@ -604,6 +604,8 @@ pub trait ReadOptions<'a> {}/// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn _get_resolved_schema<'life0, 'async_trait>(&'a self,config: &'life0 SessionConfig,@@ -662,6 +664,8 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {.with_file_sort_order(self.file_sort_order.clone())}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn get_resolved_schema<'life0, 'life1, 'async_trait>(&'life0 self,config: &'life1 SessionConfig,@@ -713,6 +717,8 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {.with_file_sort_order(self.file_sort_order.clone())}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn get_resolved_schema<'life0, 'life1, 'async_trait>(&'life0 self,config: &'life1 SessionConfig,@@ -751,6 +757,8 @@ impl ReadOptions<'_> for JsonReadOptions<'_> {.with_file_sort_order(self.file_sort_order.clone())}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn get_resolved_schema<'life0, 'life1, 'async_trait>(&'life0 self,config: &'life1 SessionConfig,@@ -785,6 +793,8 @@ impl ReadOptions<'_> for AvroReadOptions<'_> {.with_table_partition_cols(self.table_partition_cols.clone())}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn get_resolved_schema<'life0, 'life1, 'async_trait>(&'life0 self,config: &'life1 SessionConfig,@@ -818,6 +828,8 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> {.with_table_partition_cols(self.table_partition_cols.clone())}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn get_resolved_schema<'life0, 'life1, 'async_trait>(&'life0 self,config: &'life1 SessionConfig,diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 7b435822f2..f5a76c2cf5 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -48,6 +48,8 @@ pub trait ListingTableConfigExt {#[async_trait]impl ListingTableConfigExt for ListingTableConfig {+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn infer_options<'life0, 'async_trait>(self,state: &'life0 dyn Session,@@ -59,6 +61,8 @@ impl ListingTableConfigExt for ListingTableConfig {infer_options_boxed(self, state)}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn infer<'life0, 'async_trait>(self,state: &'life0 dyn Session,diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rsindex 116b8985b4..1e597e38fb 100644--- a/datafusion/core/src/datasource/listing_table_factory.rs+++ b/datafusion/core/src/datasource/listing_table_factory.rs@@ -51,6 +51,8 @@ impl ListingTableFactory {#[async_trait]impl TableProviderFactory for ListingTableFactory {+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rsindex 84bb338835..e574042813 100644--- a/datafusion/core/src/datasource/provider.rs+++ b/datafusion/core/src/datasource/provider.rs@@ -49,6 +49,8 @@ impl DefaultTableFactory {#[async_trait]impl TableProviderFactory for DefaultTableFactory {+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self,state: &'life1 dyn Session,diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex 99b16857bb..a9d6198f95 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -292,6 +292,8 @@ impl Session for SessionState {SessionState::statistics_registry(self)}+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn create_physical_plan<'life0, 'life1, 'async_trait>(&'life0 self,logical_plan: &'life1 LogicalPlan,@@ -2358,6 +2360,8 @@ struct DefaultQueryPlanner {}#[async_trait]impl QueryPlanner for DefaultQueryPlanner {/// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>(&'life0 self,logical_plan: &'life1 LogicalPlan,diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 6e50ec8aa4..77f8238edd 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -154,6 +154,8 @@ pub struct DefaultPhysicalPlanner {#[async_trait]impl PhysicalPlanner for DefaultPhysicalPlanner {/// Create a physical plan from a logical plan+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>(&'life0 self,logical_plan: &'life1 LogicalPlan,diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rsindex 036988c8fc..bcb38b97ce 100644--- a/datafusion/core/src/test_util/mod.rs+++ b/datafusion/core/src/test_util/mod.rs@@ -184,6 +184,8 @@ pub struct TestTableFactory {}#[async_trait]impl TableProviderFactory for TestTableFactory {+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self,session: &'life1 dyn Session,diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rsindex 9dbda40b9b..c1a1cd4540 100644--- a/datafusion/session/src/table.rs+++ b/datafusion/session/src/table.rs@@ -353,6 +353,8 @@ pub trait TableProvider: Any + Debug + Sync + Send {////// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64)./// Empty `filters` deletes all rows.+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn delete_from<'life0, 'life1, 'async_trait>(&'life0 self,_state: &'life1 dyn Session,@@ -373,6 +375,8 @@ pub trait TableProvider: Any + Debug + Sync + Send {////// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64)./// Empty `filters` updates all rows.+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn update<'life0, 'life1, 'async_trait>(&'life0 self,_state: &'life1 dyn Session,@@ -409,6 +413,8 @@ pub trait TableProvider: Any + Debug + Sync + Send {/// The `clauses` describe the WHEN MATCHED / WHEN NOT MATCHED actions.////// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>fn merge_into<'life0, 'life1, 'async_trait>(&'life0 self,_state: &'life1 dyn Session,