Skip to content

Apache DataFusion SQL Query Engine

Docs: document manual catalog impls and how compilation got faster (#24362)

Andrew Lamb1b67f2e9d6ad
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs
index 46fab1f9cb..498640ba0f 100644
--- a/datafusion/catalog-listing/src/table.rs
+++ b/datafusion/catalog-listing/src/table.rs
@@ -490,6 +490,8 @@ impl TableProvider for ListingTable {
TableType::Base
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -507,6 +509,8 @@ impl TableProvider for ListingTable {
self.scan_boxed(state, projection, filters, limit)
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -549,6 +553,8 @@ impl TableProvider for ListingTable {
self.definition.as_deref()
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn insert_into<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rs
index 37f66b8c56..6690fad24c 100644
--- a/datafusion/catalog/src/cte_worktable.rs
+++ b/datafusion/catalog/src/cte_worktable.rs
@@ -80,6 +80,8 @@ impl TableProvider for CteWorkTable {
TableType::Temporary
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -97,6 +99,8 @@ impl TableProvider for CteWorkTable {
self.scan_boxed(state, projection, filters, limit)
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs
index 79c6e44933..8bb45e945d 100644
--- a/datafusion/catalog/src/memory/table.rs
+++ b/datafusion/catalog/src/memory/table.rs
@@ -186,6 +186,8 @@ impl TableProvider for MemTable {
TableType::Base
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -217,6 +219,8 @@ impl TableProvider for MemTable {
/// * A plan that returns the number of rows written.
///
/// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn insert_into<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -235,6 +239,8 @@ impl TableProvider for MemTable {
self.column_defaults.get(column)
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn delete_from<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -248,6 +254,8 @@ impl TableProvider for MemTable {
self.delete_from_boxed(state, filters)
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn update<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rs
index f01874612e..86bd4e81dd 100644
--- a/datafusion/catalog/src/stream.rs
+++ b/datafusion/catalog/src/stream.rs
@@ -49,6 +49,8 @@ pub struct StreamTableFactory {}
#[async_trait]
impl TableProviderFactory for StreamTableFactory {
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn create<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -348,6 +350,8 @@ impl TableProvider for StreamTable {
TableType::Base
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -365,6 +369,8 @@ impl TableProvider for StreamTable {
self.scan_boxed(state, projection, filters, limit)
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn insert_into<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
@@ -489,6 +495,8 @@ impl DataSink for StreamWrite {
self.0.source.schema()
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn write_all<'life0, 'life1, 'async_trait>(
&'life0 self,
data: SendableRecordBatchStream,
diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rs
index bb70270ce9..7b466d8a59 100644
--- a/datafusion/catalog/src/streaming.rs
+++ b/datafusion/catalog/src/streaming.rs
@@ -123,6 +123,8 @@ impl TableProvider for StreamingTable {
TableType::View
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rs
index ac0f74c181..32af208984 100644
--- a/datafusion/catalog/src/view.rs
+++ b/datafusion/catalog/src/view.rs
@@ -96,6 +96,8 @@ impl TableProvider for ViewTable {
Ok(vec![TableProviderFilterPushDown::Exact; filters.len()])
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs
index 8be23f927e..5ca40c854b 100644
--- a/datafusion/core/src/dataframe/mod.rs
+++ b/datafusion/core/src/dataframe/mod.rs
@@ -2717,6 +2717,8 @@ impl TableProvider for DataFrameTableProvider {
self.table_type
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rs
index f670c9cc44..0212222cdd 100644
--- a/datafusion/core/src/datasource/dynamic_file.rs
+++ b/datafusion/core/src/datasource/dynamic_file.rs
@@ -54,6 +54,8 @@ impl DynamicListTableFactory {
#[async_trait]
impl UrlTableFactory for DynamicListTableFactory {
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn try_new<'life0, 'life1, 'async_trait>(
&'life0 self,
url: &'life1 str,
diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs
index 4babf4a421..f2903405c8 100644
--- a/datafusion/core/src/datasource/file_format/options.rs
+++ b/datafusion/core/src/datasource/file_format/options.rs
@@ -604,6 +604,8 @@ pub trait ReadOptions<'a> {
}
/// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn _get_resolved_schema<'life0, 'async_trait>(
&'a self,
config: &'life0 SessionConfig,
@@ -662,6 +664,8 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {
.with_file_sort_order(self.file_sort_order.clone())
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn get_resolved_schema<'life0, 'life1, 'async_trait>(
&'life0 self,
config: &'life1 SessionConfig,
@@ -713,6 +717,8 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {
.with_file_sort_order(self.file_sort_order.clone())
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn get_resolved_schema<'life0, 'life1, 'async_trait>(
&'life0 self,
config: &'life1 SessionConfig,
@@ -751,6 +757,8 @@ impl ReadOptions<'_> for JsonReadOptions<'_> {
.with_file_sort_order(self.file_sort_order.clone())
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn get_resolved_schema<'life0, 'life1, 'async_trait>(
&'life0 self,
config: &'life1 SessionConfig,
@@ -785,6 +793,8 @@ impl ReadOptions<'_> for AvroReadOptions<'_> {
.with_table_partition_cols(self.table_partition_cols.clone())
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn get_resolved_schema<'life0, 'life1, 'async_trait>(
&'life0 self,
config: &'life1 SessionConfig,
@@ -818,6 +828,8 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> {
.with_table_partition_cols(self.table_partition_cols.clone())
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn get_resolved_schema<'life0, 'life1, 'async_trait>(
&'life0 self,
config: &'life1 SessionConfig,
diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs
index 7b435822f2..f5a76c2cf5 100644
--- a/datafusion/core/src/datasource/listing/table.rs
+++ b/datafusion/core/src/datasource/listing/table.rs
@@ -48,6 +48,8 @@ pub trait ListingTableConfigExt {
#[async_trait]
impl ListingTableConfigExt for ListingTableConfig {
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn infer_options<'life0, 'async_trait>(
self,
state: &'life0 dyn Session,
@@ -59,6 +61,8 @@ impl ListingTableConfigExt for ListingTableConfig {
infer_options_boxed(self, state)
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn infer<'life0, 'async_trait>(
self,
state: &'life0 dyn Session,
diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs
index 116b8985b4..1e597e38fb 100644
--- a/datafusion/core/src/datasource/listing_table_factory.rs
+++ b/datafusion/core/src/datasource/listing_table_factory.rs
@@ -51,6 +51,8 @@ impl ListingTableFactory {
#[async_trait]
impl TableProviderFactory for ListingTableFactory {
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn create<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rs
index 84bb338835..e574042813 100644
--- a/datafusion/core/src/datasource/provider.rs
+++ b/datafusion/core/src/datasource/provider.rs
@@ -49,6 +49,8 @@ impl DefaultTableFactory {
#[async_trait]
impl TableProviderFactory for DefaultTableFactory {
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn create<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs
index 99b16857bb..a9d6198f95 100644
--- a/datafusion/core/src/execution/session_state.rs
+++ b/datafusion/core/src/execution/session_state.rs
@@ -292,6 +292,8 @@ impl Session for SessionState {
SessionState::statistics_registry(self)
}
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn create_physical_plan<'life0, 'life1, 'async_trait>(
&'life0 self,
logical_plan: &'life1 LogicalPlan,
@@ -2358,6 +2360,8 @@ struct DefaultQueryPlanner {}
#[async_trait]
impl QueryPlanner for DefaultQueryPlanner {
/// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
logical_plan: &'life1 LogicalPlan,
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index 6e50ec8aa4..77f8238edd 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -154,6 +154,8 @@ pub struct DefaultPhysicalPlanner {
#[async_trait]
impl PhysicalPlanner for DefaultPhysicalPlanner {
/// Create a physical plan from a logical plan
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
logical_plan: &'life1 LogicalPlan,
diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs
index 036988c8fc..bcb38b97ce 100644
--- a/datafusion/core/src/test_util/mod.rs
+++ b/datafusion/core/src/test_util/mod.rs
@@ -184,6 +184,8 @@ pub struct TestTableFactory {}
#[async_trait]
impl TableProviderFactory for TestTableFactory {
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn create<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
session: &'life1 dyn Session,
diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rs
index 9dbda40b9b..c1a1cd4540 100644
--- a/datafusion/session/src/table.rs
+++ b/datafusion/session/src/table.rs
@@ -353,6 +353,8 @@ pub trait TableProvider: Any + Debug + Sync + Send {
///
/// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).
/// Empty `filters` deletes all rows.
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn delete_from<'life0, 'life1, 'async_trait>(
&'life0 self,
_state: &'life1 dyn Session,
@@ -373,6 +375,8 @@ pub trait TableProvider: Any + Debug + Sync + Send {
///
/// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).
/// Empty `filters` updates all rows.
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn update<'life0, 'life1, 'async_trait>(
&'life0 self,
_state: &'life1 dyn Session,
@@ -409,6 +413,8 @@ pub trait TableProvider: Any + Debug + Sync + Send {
/// The `clauses` describe the WHEN MATCHED / WHEN NOT MATCHED actions.
///
/// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).
+ // Hand-written `#[async_trait]` expansion to reduce compile time. See
+ // <https://github.com/apache/datafusion/issues/13814#issuecomment-5292709677>
fn merge_into<'life0, 'life1, 'async_trait>(
&'life0 self,
_state: &'life1 dyn Session,