Apache DataFusion SQL Query Engine
perf: Update `TableProvider` to take scan projection as `Option<&[usize]>` (#24376)
Daniël Heresb9399dc22869
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rsindex 164af2559d..409eedee75 100644--- a/datafusion-cli/src/functions.rs+++ b/datafusion-cli/src/functions.rs@@ -241,14 +241,14 @@ impl TableProvider for ParquetMetadataTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {Ok(MemorySourceConfig::try_new_exec(&[vec![self.batch.clone()]],TableProvider::schema(self),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?)}}@@ -487,14 +487,14 @@ impl TableProvider for MetadataCacheTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {Ok(MemorySourceConfig::try_new_exec(&[vec![self.batch.clone()]],TableProvider::schema(self),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?)}}@@ -604,14 +604,14 @@ impl TableProvider for StatisticsCacheTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {Ok(MemorySourceConfig::try_new_exec(&[vec![self.batch.clone()]],TableProvider::schema(self),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?)}}@@ -753,14 +753,14 @@ impl TableProvider for ListFilesCacheTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {Ok(MemorySourceConfig::try_new_exec(&[vec![self.batch.clone()]],TableProvider::schema(self),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?)}}diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rsindex 6f176f8b46..91fb219f34 100644--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs@@ -148,7 +148,7 @@ impl Debug for CustomDataSource {impl CustomDataSource {pub(crate) fn create_physical_plan(&self,- projections: Option<&Vec<usize>>,+ projections: Option<&[usize]>,schema: SchemaRef,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(CustomExec::new(projections, schema, self.clone())))@@ -203,7 +203,7 @@ impl TableProvider for CustomDataSource {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,// filters and limit can be used here to inject some push-down operations if needed_filters: &[Expr],_limit: Option<usize>,@@ -223,7 +223,7 @@ struct CustomExec {impl CustomExec {#[expect(clippy::needless_pass_by_value)]fn new(- projections: Option<&Vec<usize>>,+ projections: Option<&[usize]>,schema: SchemaRef,db: CustomDataSource,) -> Self {@@ -231,7 +231,7 @@ impl CustomExec {let cache = Self::compute_properties(projected_schema.clone());Self {db,- projection: projections.cloned(),+ projection: projections.map(|p| p.to_vec()),projected_schema,cache: Arc::new(cache),}diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rsindex 633b982443..d2024621aa 100644--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs@@ -218,7 +218,7 @@ impl TableProvider for DefaultValueTableProvider {async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -253,7 +253,7 @@ impl TableProvider for DefaultValueTableProvider {ObjectStoreUrl::parse("memory://")?,Arc::new(parquet_source),)- .with_projection_indices(projection.cloned())?+ .with_projection_indices(projection.map(|p| p.to_vec()))?.with_limit(limit).with_file_group(file_group).with_expr_adapter(Some(Arc::new(DefaultValuePhysicalExprAdapterFactory) as _));diff --git a/datafusion-examples/examples/data_io/parquet_advanced_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rsindex b6440eb3e2..47c57a4804 100644--- a/datafusion-examples/examples/data_io/parquet_advanced_index.rs+++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs@@ -464,7 +464,7 @@ impl TableProvider for IndexTableProvider {async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -500,7 +500,7 @@ impl TableProvider for IndexTableProvider {);let file_scan_config = FileScanConfigBuilder::new(object_store_url, file_source).with_limit(limit)- .with_projection_indices(projection.cloned())?+ .with_projection_indices(projection.map(|p| p.to_vec()))?.with_file(partitioned_file).build();diff --git a/datafusion-examples/examples/data_io/parquet_embedded_index.rs b/datafusion-examples/examples/data_io/parquet_embedded_index.rsindex a8a3c97fa1..5e48600650 100644--- a/datafusion-examples/examples/data_io/parquet_embedded_index.rs+++ b/datafusion-examples/examples/data_io/parquet_embedded_index.rs@@ -405,7 +405,7 @@ impl TableProvider for DistinctIndexTable {async fn scan(&self,_ctx: &dyn Session,- _proj: Option<&Vec<usize>>,+ _proj: Option<&[usize]>,filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion-examples/examples/data_io/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rsindex 753d1b30fc..c8ae7e14c4 100644--- a/datafusion-examples/examples/data_io/parquet_index.rs+++ b/datafusion-examples/examples/data_io/parquet_index.rs@@ -218,7 +218,7 @@ impl TableProvider for IndexTableProvider {async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -242,7 +242,7 @@ impl TableProvider for IndexTableProvider {Arc::new(ParquetSource::new(self.schema()).with_predicate(predicate));let mut file_scan_config_builder =FileScanConfigBuilder::new(object_store_url, source)- .with_projection_indices(projection.cloned())?+ .with_projection_indices(projection.map(|p| p.to_vec()))?.with_limit(limit);// Transform to the format needed to pass to DataSourceExecdiff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rsindex a24ca22381..49157c6b1f 100644--- a/datafusion-examples/examples/data_io/remote_catalog.rs+++ b/datafusion-examples/examples/data_io/remote_catalog.rs@@ -237,7 +237,7 @@ impl TableProvider for RemoteTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -257,7 +257,7 @@ impl TableProvider for RemoteTable {let exec = MemorySourceConfig::try_new_exec(&[batches],self.schema.clone(),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?;Ok(exec)}diff --git a/datafusion-examples/examples/udf/simple_udtf.rs b/datafusion-examples/examples/udf/simple_udtf.rsindex af123ab7e5..3b55a0456a 100644--- a/datafusion-examples/examples/udf/simple_udtf.rs+++ b/datafusion-examples/examples/udf/simple_udtf.rs@@ -96,7 +96,7 @@ impl TableProvider for LocalCsvTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -122,7 +122,7 @@ impl TableProvider for LocalCsvTable {Ok(MemorySourceConfig::try_new_exec(&[batches],TableProvider::schema(self),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?)}}diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 498640ba0f..1749cdac2f 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -495,7 +495,7 @@ impl TableProvider for ListingTable {fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,- projection: Option<&'life2 Vec<usize>>,+ projection: Option<&'life2 [usize]>,filters: &'life3 [Expr],limit: Option<usize>,) -> BoxFuture<'async_trait, datafusion_common::Result<Arc<dyn ExecutionPlan>>>@@ -753,7 +753,7 @@ impl ListingTable {fn scan_boxed<'a>(&'a self,state: &'a dyn Session,- projection: Option<&'a Vec<usize>>,+ projection: Option<&'a [usize]>,filters: &'a [Expr],limit: Option<usize>,) -> BoxFuture<'a, datafusion_common::Result<Arc<dyn ExecutionPlan>>> {@@ -763,12 +763,12 @@ impl ListingTable {async fn scan_inner(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {let options = ScanArgs::default()- .with_projection(projection.map(|p| p.as_slice()))+ .with_projection(projection).with_filters(Some(filters)).with_limit(limit);Ok(self.scan_with_args(state, options).await?.into_inner())diff --git a/datafusion/catalog/src/async.rs b/datafusion/catalog/src/async.rsindex 87b7b7c343..6f8e25cafd 100644--- a/datafusion/catalog/src/async.rs+++ b/datafusion/catalog/src/async.rs@@ -444,7 +444,7 @@ mod tests {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rsindex 6690fad24c..5f5094793c 100644--- a/datafusion/catalog/src/cte_worktable.rs+++ b/datafusion/catalog/src/cte_worktable.rs@@ -85,7 +85,7 @@ impl TableProvider for CteWorkTable {fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,- projection: Option<&'life2 Vec<usize>>,+ projection: Option<&'life2 [usize]>,filters: &'life3 [Expr],limit: Option<usize>,) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>@@ -143,7 +143,7 @@ impl CteWorkTable {fn scan_boxed<'a>(&'a self,state: &'a dyn Session,- projection: Option<&'a Vec<usize>>,+ projection: Option<&'a [usize]>,filters: &'a [Expr],limit: Option<usize>,) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {@@ -153,12 +153,12 @@ impl CteWorkTable {async fn scan_inner(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {let options = ScanArgs::default()- .with_projection(projection.map(|p| p.as_slice()))+ .with_projection(projection).with_filters(Some(filters)).with_limit(limit);Ok(self.scan_with_args(state, options).await?.into_inner())diff --git a/datafusion/catalog/src/default_table_source.rs b/datafusion/catalog/src/default_table_source.rsindex 60f85891d6..3342db54de 100644--- a/datafusion/catalog/src/default_table_source.rs+++ b/datafusion/catalog/src/default_table_source.rs@@ -118,7 +118,7 @@ fn preserves_table_type() {async fn scan(&self,_: &dyn crate::Session,- _: Option<&Vec<usize>>,+ _: Option<&[usize]>,_: &[Expr],_: Option<usize>,) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>, DataFusionError>diff --git a/datafusion/catalog/src/empty.rs b/datafusion/catalog/src/empty.rsindex 1ff36ecf36..66b62bfcec 100644--- a/datafusion/catalog/src/empty.rs+++ b/datafusion/catalog/src/empty.rs@@ -66,7 +66,7 @@ impl TableProvider for EmptyTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 8bb45e945d..d817aa8b77 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -191,7 +191,7 @@ impl TableProvider for MemTable {fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,- projection: Option<&'life2 Vec<usize>>,+ projection: Option<&'life2 [usize]>,filters: &'life3 [Expr],limit: Option<usize>,) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>@@ -275,7 +275,7 @@ impl MemTable {fn scan_boxed<'a>(&'a self,state: &'a dyn Session,- projection: Option<&'a Vec<usize>>,+ projection: Option<&'a [usize]>,filters: &'a [Expr],limit: Option<usize>,) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {@@ -285,7 +285,7 @@ impl MemTable {async fn scan_inner(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -295,8 +295,11 @@ impl MemTable {partitions.push(inner_vec.clone())}- let mut source =- MemorySourceConfig::try_new(&partitions, self.schema(), projection.cloned())?;+ let mut source = MemorySourceConfig::try_new(+ &partitions,+ self.schema(),+ projection.map(|p| p.to_vec()),+ )?;let show_sizes = state.config_options().explain.show_sizes;source = source.with_show_sizes(show_sizes);diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rsindex 86bd4e81dd..97e5293765 100644--- a/datafusion/catalog/src/stream.rs+++ b/datafusion/catalog/src/stream.rs@@ -355,7 +355,7 @@ impl TableProvider for StreamTable {fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,- projection: Option<&'life2 Vec<usize>>,+ projection: Option<&'life2 [usize]>,filters: &'life3 [Expr],limit: Option<usize>,) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>@@ -390,7 +390,7 @@ impl StreamTable {fn scan_boxed<'a>(&'a self,state: &'a dyn Session,- projection: Option<&'a Vec<usize>>,+ projection: Option<&'a [usize]>,filters: &'a [Expr],limit: Option<usize>,) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {@@ -400,7 +400,7 @@ impl StreamTable {fn scan_inner(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rsindex 7b466d8a59..c2b5d69186 100644--- a/datafusion/catalog/src/streaming.rs+++ b/datafusion/catalog/src/streaming.rs@@ -95,10 +95,7 @@ impl StreamingTable {self}- fn output_partitioning(- &self,- projection: Option<&Vec<usize>>,- ) -> Result<Partitioning> {+ fn output_partitioning(&self, projection: Option<&[usize]>) -> Result<Partitioning> {let Some(output_partitioning) = &self.output_partitioning else {return Ok(Partitioning::UnknownPartitioning(self.partitions.len()));};@@ -128,7 +125,7 @@ impl TableProvider for StreamingTable {fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,- projection: Option<&'life2 Vec<usize>>,+ projection: Option<&'life2 [usize]>,filters: &'life3 [Expr],limit: Option<usize>,) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>@@ -147,7 +144,7 @@ impl StreamingTable {fn scan_boxed<'a>(&'a self,state: &'a dyn Session,- projection: Option<&'a Vec<usize>>,+ projection: Option<&'a [usize]>,filters: &'a [Expr],limit: Option<usize>,) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {@@ -157,7 +154,7 @@ impl StreamingTable {fn scan_inner(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rsindex 32af208984..795d2985c8 100644--- a/datafusion/catalog/src/view.rs+++ b/datafusion/catalog/src/view.rs@@ -101,7 +101,7 @@ impl TableProvider for ViewTable {fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,- projection: Option<&'life2 Vec<usize>>,+ projection: Option<&'life2 [usize]>,filters: &'life3 [Expr],limit: Option<usize>,) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>@@ -120,7 +120,7 @@ impl ViewTable {fn scan_boxed<'a>(&'a self,state: &'a dyn Session,- projection: Option<&'a Vec<usize>>,+ projection: Option<&'a [usize]>,filters: &'a [Expr],limit: Option<usize>,) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {@@ -130,7 +130,7 @@ impl ViewTable {async fn scan_inner(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -146,7 +146,7 @@ impl ViewTable {// avoiding adding a redundant projection (e.g. SELECT * FROM view)let current_projection =(0..plan.schema().fields().len()).collect::<Vec<usize>>();- if projection == ¤t_projection {+ if projection == current_projection {plan} else {let fields: Vec<Expr> = projectiondiff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex 73772b3193..8df160187a 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -80,9 +80,9 @@ use std::thread::available_parallelism;////// assert_eq!(projected_schema, expected_schema);/// ```-pub fn project_schema(+pub fn project_schema<T: AsRef<[usize]> + ?Sized>(schema: &SchemaRef,- projection: Option<&impl AsRef<[usize]>>,+ projection: Option<&T>,) -> Result<SchemaRef> {let schema = match projection {Some(columns) => Arc::new(schema.project(columns.as_ref())?),diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 5ca40c854b..333f94dc8f 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -2722,7 +2722,7 @@ impl TableProvider for DataFrameTableProvider {fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self,state: &'life1 dyn Session,- projection: Option<&'life2 Vec<usize>>,+ projection: Option<&'life2 [usize]>,filters: &'life3 [Expr],limit: Option<usize>,) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>@@ -2741,7 +2741,7 @@ impl DataFrameTableProvider {fn scan_boxed<'a>(&'a self,state: &'a dyn Session,- projection: Option<&'a Vec<usize>>,+ projection: Option<&'a [usize]>,filters: &'a [Expr],limit: Option<usize>,) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {@@ -2751,7 +2751,7 @@ impl DataFrameTableProvider {async fn scan_inner(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/core/src/datasource/memory_test.rs b/datafusion/core/src/datasource/memory_test.rsindex d7311c1d9c..9e37f77d8b 100644--- a/datafusion/core/src/datasource/memory_test.rs+++ b/datafusion/core/src/datasource/memory_test.rs@@ -60,7 +60,7 @@ mod tests {// scan with projectionlet exec = provider- .scan(&session_ctx.state(), Some(&vec![2, 1]), &[], None)+ .scan(&session_ctx.state(), Some(&[2, 1]), &[], None).await?;let mut it = exec.execute(0, task_ctx)?;diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 77f8238edd..7002a4b04d 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -3538,7 +3538,7 @@ mod tests {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -5526,7 +5526,7 @@ digraph {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rsindex bcb38b97ce..ab7edef885 100644--- a/datafusion/core/src/test_util/mod.rs+++ b/datafusion/core/src/test_util/mod.rs@@ -250,7 +250,7 @@ impl TableProvider for TestTableProvider {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/core/tests/custom_sources_cases/dml_planning.rs b/datafusion/core/tests/custom_sources_cases/dml_planning.rsindex 24a3df7e0a..6ae942bfbb 100644--- a/datafusion/core/tests/custom_sources_cases/dml_planning.rs+++ b/datafusion/core/tests/custom_sources_cases/dml_planning.rs@@ -101,7 +101,7 @@ impl TableProvider for CaptureDeleteProvider {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -197,7 +197,7 @@ impl TableProvider for CaptureUpdateProvider {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -271,7 +271,7 @@ impl TableProvider for CaptureTruncateProvider {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rsindex 7abbcd6e95..ea3cdaa34d 100644--- a/datafusion/core/tests/custom_sources_cases/mod.rs+++ b/datafusion/core/tests/custom_sources_cases/mod.rs@@ -247,11 +247,13 @@ impl TableProvider for CustomTableProvider {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(CustomExecutionPlan::new(projection.cloned())))+ Ok(Arc::new(CustomExecutionPlan::new(+ projection.map(|p| p.to_vec()),+ )))}}diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rsindex a8f7f09ad0..51d31360f6 100644--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs@@ -191,7 +191,7 @@ impl TableProvider for CustomProvider {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],_: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rsindex 6213be8e2d..7701ca0279 100644--- a/datafusion/core/tests/custom_sources_cases/statistics.rs+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs@@ -91,7 +91,7 @@ impl TableProvider for StatisticsValidation {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],// limit is ignored because it is not mandatory for a `TableProvider` to honor it_limit: Option<usize>,@@ -102,7 +102,7 @@ impl TableProvider for StatisticsValidation {filters.len(),"Unsupported expressions should not be pushed down");- let projection = match projection.cloned() {+ let projection = match projection.map(|p| p.to_vec()) {Some(p) => p,None => (0..self.schema.fields().len()).collect(),};diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rsindex 84d7e9c450..472ff07aac 100644--- a/datafusion/core/tests/memory_limit/mod.rs+++ b/datafusion/core/tests/memory_limit/mod.rs@@ -1216,14 +1216,14 @@ impl TableProvider for SortedTableProvider {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {let mem_conf = MemorySourceConfig::try_new(&self.batches,self.schema(),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?.try_with_sort_information(self.sort_information.clone())?;diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rsindex 3a8d82f111..113552c462 100644--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs@@ -561,7 +561,7 @@ fn test_streaming_table_after_projection() -> Result<()> {Field::new("e", DataType::Int32, true),])),}) as _],- Some(&vec![0_usize, 2, 4, 3]),+ Some(&[0_usize, 2, 4, 3]),vec![[PhysicalSortExpr {diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rsindex 0eefcdb551..49d0ecc1ae 100644--- a/datafusion/core/tests/user_defined/insert_operation.rs+++ b/datafusion/core/tests/user_defined/insert_operation.rs@@ -101,7 +101,7 @@ impl TableProvider for TestInsertTableProvider {async fn scan(&self,_state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/core/tests/user_defined/statistics_requests.rs b/datafusion/core/tests/user_defined/statistics_requests.rsindex 64c6676c23..079639baf1 100644--- a/datafusion/core/tests/user_defined/statistics_requests.rs+++ b/datafusion/core/tests/user_defined/statistics_requests.rs@@ -113,14 +113,14 @@ impl TableProvider for RecordingTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {Ok(MemorySourceConfig::try_new_exec(&[vec![self.batch.clone()]],Arc::clone(&self.schema),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?)}@@ -134,7 +134,7 @@ impl TableProvider for RecordingTable {let plan = self.scan(state,- args.projection().map(|p| p.to_vec()).as_ref(),+ args.projection(),args.filters().unwrap_or(&[]),args.limit(),)diff --git a/datafusion/core/tests/user_defined/user_defined_table_functions.rs b/datafusion/core/tests/user_defined/user_defined_table_functions.rsindex c8ded3a6fc..24205cf8c4 100644--- a/datafusion/core/tests/user_defined/user_defined_table_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_table_functions.rs@@ -129,7 +129,7 @@ impl TableProvider for SimpleCsvTable {async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -156,7 +156,7 @@ impl TableProvider for SimpleCsvTable {Ok(MemorySourceConfig::try_new_exec(&[batches],TableProvider::schema(self),- projection.cloned(),+ projection.map(|p| p.to_vec()),)?)}}diff --git a/datafusion/ffi/src/table_provider.rs b/datafusion/ffi/src/table_provider.rsindex f1a79added..b1d54b5f33 100644--- a/datafusion/ffi/src/table_provider.rs+++ b/datafusion/ffi/src/table_provider.rs@@ -320,7 +320,7 @@ unsafe extern "C" fn scan_fn_wrapper(let plan = sresult_return!(internal_provider- .scan(session, projections.as_ref(), &filters, limit.into())+ .scan(session, projections.as_deref(), &filters, limit.into()).await);@@ -653,7 +653,7 @@ impl TableProvider for ForeignTableProvider {async fn scan(&self,session: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -874,7 +874,7 @@ mod tests {async fn scan(&self,session: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rsindex 83057d8c45..cb5d27c6e6 100644--- a/datafusion/ffi/src/tests/async_provider.rs+++ b/datafusion/ffi/src/tests/async_provider.rs@@ -139,7 +139,7 @@ impl TableProvider for AsyncTableProvider {async fn scan(&self,state: &dyn Session,- _projection: Option<&Vec<usize>>,+ _projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rsindex ce922de664..fbc3e83ba4 100644--- a/datafusion/ffi/src/tests/mod.rs+++ b/datafusion/ffi/src/tests/mod.rs@@ -259,7 +259,7 @@ impl TableProvider for TableWithStats {async fn scan(&self,session: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/ffi/src/tests/query_planner.rs b/datafusion/ffi/src/tests/query_planner.rsindex 90d713f8a6..40c237c39e 100644--- a/datafusion/ffi/src/tests/query_planner.rs+++ b/datafusion/ffi/src/tests/query_planner.rs@@ -57,7 +57,12 @@ impl QueryPlanner for TestQueryPlanner {return exec_err!("library B's provider was not foreign to library C");}let library_b_plan = provider- .scan(session, scan.projection.as_ref(), &scan.filters, scan.fetch)+ .scan(+ session,+ scan.projection.as_deref(),+ &scan.filters,+ scan.fetch,+ ).await?;if !library_b_plan.is::<ForeignExecutionPlan>() {diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rsindex f5e4df1389..8d9327c5ac 100644--- a/datafusion/functions-table/src/generate_series.rs+++ b/datafusion/functions-table/src/generate_series.rs@@ -552,14 +552,14 @@ impl TableProvider for GenerateSeriesTable {async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],_limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {let batch_size = state.config_options().execution.batch_size.get();let generator = self.as_generator(batch_size)?;let mut exec = LazyMemoryExec::try_new(self.schema(), vec![generator])?- .with_projection(projection.cloned());+ .with_projection(projection.map(|p| p.to_vec()));if let Some(ordering) = self.output_ordering(exec.schema().as_ref()) {exec.add_ordering([ordering]);diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 5df5482fb7..c62db109ea 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -1236,7 +1236,7 @@ pub fn batch_filter(fn filter_and_project(batch: &RecordBatch,predicate: &Arc<dyn PhysicalExpr>,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,) -> Result<RecordBatch> {predicate.evaluate(batch)diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rsindex 7b0058e798..dd57f958e8 100644--- a/datafusion/physical-plan/src/streaming.rs+++ b/datafusion/physical-plan/src/streaming.rs@@ -81,7 +81,7 @@ impl StreamingTableExec {pub fn try_new(schema: SchemaRef,partitions: Vec<Arc<dyn PartitionStream>>,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,projected_output_ordering: impl IntoIterator<Item = LexOrdering>,infinite: bool,limit: Option<usize>,@@ -112,7 +112,7 @@ impl StreamingTableExec {Ok(Self {partitions,projected_schema,- projection: projection.cloned().map(Into::into),+ projection: projection.map(|p| p.to_vec().into()),projected_output_ordering,infinite,limit,@@ -475,7 +475,7 @@ mod test {StreamingTableExec::try_new(self.schema.unwrap(),self.partitions,- self.projection.as_ref(),+ self.projection.as_deref(),self.projected_output_ordering,self.infinite,self.limit,diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rsindex c1a1cd4540..a6cddd10ce 100644--- a/datafusion/session/src/table.rs+++ b/datafusion/session/src/table.rs@@ -187,7 +187,7 @@ pub trait TableProvider: Any + Debug + Sync + Send {async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>>;@@ -209,18 +209,26 @@ pub trait TableProvider: Any + Debug + Sync + Send {/// A [`ScanResult`] containing the [`ExecutionPlan`] for scanning the table////// See [`Self::scan`] for detailed documentation about projection, filters, and limits.- async fn scan_with_args<'a>(- &self,- state: &dyn Session,+ // Hand-written `#[async_trait]` expansion to reduce compile time. See+ // <https://github.com/apache/datafusion/issues/13814>.+ fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(+ &'life0 self,+ state: &'life1 dyn Session,args: ScanArgs<'a>,- ) -> Result<ScanResult> {- let filters = args.filters().unwrap_or(&[]);- let projection = args.projection().map(|p| p.to_vec());- let limit = args.limit();- let plan = self- .scan(state, projection.as_ref(), filters, limit)- .await?;- Ok(plan.into())+ ) -> BoxFuture<'async_trait, Result<ScanResult>>+ where+ 'a: 'async_trait,+ 'life0: 'async_trait,+ 'life1: 'async_trait,+ Self: 'async_trait,+ {+ let plan = self.scan(+ state,+ args.projection(),+ args.filters().unwrap_or(&[]),+ args.limit(),+ );+ Box::pin(async move { Ok(plan.await?.into()) })}/// Specify if DataFusion should provide filter expressions to the@@ -269,7 +277,7 @@ pub trait TableProvider: Any + Debug + Sync + Send {/// impl TableProvider for TestDataSource {/// # fn schema(&self) -> SchemaRef { todo!() }/// # fn table_type(&self) -> TableType { todo!() }- /// # async fn scan(&self, s: &dyn Session, p: Option<&Vec<usize>>, f: &[Expr], l: Option<usize>) -> Result<Arc<dyn ExecutionPlan>> {+ /// # async fn scan(&self, s: &dyn Session, p: Option<&[usize]>, f: &[Expr], l: Option<usize>) -> Result<Arc<dyn ExecutionPlan>> {/// todo!()/// # }/// // Override the supports_filters_pushdown to evaluate which expressionsdiff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex d85ca2db76..39aa2b09e5 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -388,7 +388,7 @@ pub fn register_temp_table(ctx: &SessionContext) {async fn scan(&self,_state: &dyn Session,- _: Option<&Vec<usize>>,+ _: Option<&[usize]>,_: &[Expr],_: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>, DataFusionError> {diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.mdindex c094f8bf7e..dbbdf9d022 100644--- a/docs/source/library-user-guide/custom-table-providers.md+++ b/docs/source/library-user-guide/custom-table-providers.md@@ -170,7 +170,7 @@ impl TableProvider for MyTable {async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -323,7 +323,7 @@ expects to work with:async fn scan(&self,state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -528,7 +528,7 @@ To opt in, implement `supports_filters_pushdown`:# impl TableProvider for MyFilterTable {# fn schema(&self) -> SchemaRef { todo!() }# fn table_type(&self) -> TableType { TableType::Base }-# async fn scan(&self, _: &dyn Session, _: Option<&Vec<usize>>, _: &[Expr], _: Option<usize>) -> Result<Arc<dyn ExecutionPlan>> { todo!() }+# async fn scan(&self, _: &dyn Session, _: Option<&[usize]>, _: &[Expr], _: Option<usize>) -> Result<Arc<dyn ExecutionPlan>> { todo!() }#fn supports_filters_pushdown(&self,@@ -700,7 +700,7 @@ impl TableProvider for DatePartitionedTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -848,7 +848,7 @@ impl TableProvider for CountingTable {async fn scan(&self,_state: &dyn Session,- projection: Option<&Vec<usize>>,+ projection: Option<&[usize]>,_filters: &[Expr],limit: Option<usize>,) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex d64f287ea0..21def5a1fc 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -25,6 +25,57 @@in this section pertains to features and changes that have already been mergedto the main branch and are awaiting release in this version.+### `TableProvider::scan` takes `Option<&[usize]>` for the projection++`TableProvider::scan` and the related APIs listed below previously took the+projection as `Option<&Vec<usize>>`:++```rust,ignore+// Before+async fn scan(+ &self,+ state: &dyn Session,+ projection: Option<&Vec<usize>>,+ filters: &[Expr],+ limit: Option<usize>,+) -> Result<Arc<dyn ExecutionPlan>>;++// After+async fn scan(+ &self,+ state: &dyn Session,+ projection: Option<&[usize]>,+ filters: &[Expr],+ limit: Option<usize>,+) -> Result<Arc<dyn ExecutionPlan>>;+```++Implementations only need the signature updated. Two patterns inside a `scan`+body need adjusting, because `Option<&[usize]>` is not an `Option<&T>` over a+sized type:++```rust,ignore+// Before // After+projection.cloned() projection.map(|p| p.to_vec())+```++Callers holding an `Option<Vec<usize>>` should pass `.as_deref()` rather than+`.as_ref()`.++The same change applies to `StreamingTableExec::try_new`,+`FilterExec::with_projection` and `batch_filter`, which took the projection in+the same shape.++`datafusion_common::project_schema` now accepts unsized projections+(`projection: Option<&T> where T: AsRef<[usize]> + ?Sized`), so it takes both+`Option<&Vec<usize>>` and `Option<&[usize]>`; no caller changes are required.++Besides being the more idiomatic signature, this lets+`TableProvider::scan_with_args`'s default body hand the projection straight to+`scan` instead of allocating a `Vec` that then has to be captured by the+returned future -- which is what made that method expensive to compile+(`datafusion-session` compiles ~3.3x faster as a result).+### `DataFrame::fill_null` now borrows its arguments`DataFrame::fill_null` previously took its arguments by value: