Apache DataFusion SQL Query Engine
Better document the relationship between `FileFormat::projection` / `FileFormat::filter` and `FileScanConfig::Statistics` (#20188)
Andrew Lambcc670e8e88af
diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex c6282c3c7c..b41a456f1f 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -46,6 +46,12 @@ pub fn as_file_source<T: FileSource + 'static>(source: T) -> Arc<dyn FileSource>/// file format specific behaviors for elements in [`DataSource`]///+/// # Schema information+/// There are two important schemas for a [`FileSource`]:+/// 1. [`Self::table_schema`] -- the schema for the overall "table"+/// 2. The logical output schema, comprised of [`Self::table_schema`] with+/// [`Self::projection`] applied+////// See more details on specific implementations:/// * [`ArrowSource`](https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.ArrowSource.html)/// * [`AvroSource`](https://docs.rs/datafusion/latest/datafusion/datasource/physical_plan/struct.AvroSource.html)@@ -64,24 +70,38 @@ pub trait FileSource: Send + Sync {) -> Result<Arc<dyn FileOpener>>;/// Anyfn as_any(&self) -> &dyn Any;+/// Returns the table schema for this file source.///- /// This always returns the unprojected schema (the full schema of the data).+ /// This always returns the unprojected schema (the full schema of the data)+ /// without [`Self::projection`] applied.+ ///+ /// The output schema of this `FileSource` is this TableSchema+ /// with [`Self::projection`] applied.fn table_schema(&self) -> &crate::table_schema::TableSchema;+/// Initialize new type with batch size configurationfn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>;- /// Returns the filter expression that will be applied during the file scan.++ /// Returns the filter expression that will be applied *during* the file scan.+ ///+ /// These expressions are in terms of the unprojected [`Self::table_schema`].fn filter(&self) -> Option<Arc<dyn PhysicalExpr>> {None}- /// Return the projection that will be applied to the output stream on top of the table schema.++ /// Return the projection that will be applied to the output stream on top+ /// of [`Self::table_schema`].fn projection(&self) -> Option<&ProjectionExprs> {None}+/// Return execution plan metricsfn metrics(&self) -> &ExecutionPlanMetricsSet;+/// String representation of file source such as "csv", "json", "parquet"fn file_type(&self) -> &str;+/// Format FileType specific informationfn fmt_extra(&self, _t: DisplayFormatType, _f: &mut Formatter) -> fmt::Result {Ok(())@@ -135,6 +155,19 @@ pub trait FileSource: Send + Sync {}/// Try to push down filters into this FileSource.+ ///+ /// `filters` must be in terms of the unprojected table schema (file schema+ /// plus partition columns), before any projection is applied.+ ///+ /// Any filters that this FileSource chooses to evaluate itself should be+ /// returned as `PushedDown::Yes` in the result, along with a FileSource+ /// instance that incorporates those filters. Such filters are logically+ /// applied "during" the file scan, meaning they may refer to columns not+ /// included in the final output projection.+ ///+ /// Filters that cannot be pushed down should be marked as `PushedDown::No`,+ /// and will be evaluated by an execution plan after the file source.+ ////// See [`ExecutionPlan::handle_child_pushdown_result`] for more details.////// [`ExecutionPlan::handle_child_pushdown_result`]: datafusion_physical_plan::ExecutionPlan::handle_child_pushdown_result@@ -220,7 +253,7 @@ pub trait FileSource: Send + Sync {Ok(SortOrderPushdownResult::Unsupported)}- /// Try to push down a projection into a this FileSource.+ /// Try to push down a projection into this FileSource.////// `FileSource` implementations that support projection pushdown should/// override this method and return a new `FileSource` instance with thediff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex fe78c0e526..80722755e6 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -55,10 +55,21 @@ use datafusion_physical_plan::{use log::{debug, warn};use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};-/// The base configurations for a [`DataSourceExec`], the a physical plan for-/// any given file format.+/// [`FileScanConfig`] represents scanning data from a group of files///-/// Use [`DataSourceExec::from_data_source`] to create a [`DataSourceExec`] from a ``FileScanConfig`.+/// `FileScanConfig` is used to create a [`DataSourceExec`], the physical plan+/// for scanning files with a particular file format.+///+/// The [`FileSource`] (e.g. `ParquetSource`, `CsvSource`, etc.) is responsible+/// for creating the actual execution plan to read the files based on a+/// `FileScanConfig`. Fields in a `FileScanConfig` such as Statistics represent+/// information about the files **before** any projection or filtering is+/// applied in the file source.+///+/// Use [`FileScanConfigBuilder`] to construct a `FileScanConfig`.+///+/// Use [`DataSourceExec::from_data_source`] to create a [`DataSourceExec`] from+/// a `FileScanConfig`.////// # Example/// ```@@ -169,8 +180,11 @@ pub struct FileScanConfig {/// Expression adapter used to adapt filters and projections that are pushed down into the scan/// from the logical schema to the physical schema of the file.pub expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,- /// Unprojected statistics for the table (file schema + partition columns).- /// These are projected on-demand via `projected_stats()`.+ /// Statistics for the entire table (file schema + partition columns).+ /// See [`FileScanConfigBuilder::with_statistics`] for more details.+ ///+ /// The effective statistics are computed on-demand via+ /// [`ProjectionExprs::project_statistics`].////// Note that this field is pub(crate) because accessing it directly from outside/// would be incorrect if there are filters being applied, thus this should be accessed@@ -283,17 +297,20 @@ impl FileScanConfigBuilder {}}- /// Set the maximum number of records to read from this plan. If `None`,- /// all records after filtering are returned.+ /// Set the maximum number of records to read from this plan.+ ///+ /// If `None`, all records after filtering are returned.pub fn with_limit(mut self, limit: Option<usize>) -> Self {self.limit = limit;self}/// Set whether the limit should be order-sensitive.+ ////// When `true`, files must be read in the exact order specified to produce/// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,- /// DataFusion may reorder file processing for optimization without affecting correctness.+ /// DataFusion may reorder file processing for optimization without+ /// affecting correctness.pub fn with_preserve_order(mut self, order_sensitive: bool) -> Self {self.preserve_order = order_sensitive;self@@ -301,13 +318,14 @@ impl FileScanConfigBuilder {/// Set the file source for scanning files.///- /// This method allows you to change the file source implementation (e.g. ParquetSource, CsvSource, etc.)- /// after the builder has been created.+ /// This method allows you to change the file source implementation (e.g.+ /// ParquetSource, CsvSource, etc.) after the builder has been created.pub fn with_source(mut self, file_source: Arc<dyn FileSource>) -> Self {self.file_source = file_source;self}+ /// Return the table schemapub fn table_schema(&self) -> &SchemaRef {self.file_source.table_schema().table_schema()}@@ -332,7 +350,12 @@ impl FileScanConfigBuilder {/// Set the columns on which to project the data using column indices.///- /// Indexes that are higher than the number of columns of `file_schema` refer to `table_partition_cols`.+ /// This method attempts to push down the projection to the underlying file+ /// source if supported. If the file source does not support projection+ /// pushdown, an error is returned.+ ///+ /// Indexes that are higher than the number of columns of `file_schema`+ /// refer to `table_partition_cols`.pub fn with_projection_indices(mut self,indices: Option<Vec<usize>>,@@ -371,8 +394,18 @@ impl FileScanConfigBuilder {self}- /// Set the estimated overall statistics of the files, taking `filters` into account.- /// Defaults to [`Statistics::new_unknown`].+ /// Set the statistics of the files, including partition+ /// columns. Defaults to [`Statistics::new_unknown`].+ ///+ /// These statistics are for the entire table (file schema + partition+ /// columns) before any projection or filtering is applied. Projections are+ /// applied when statistics are retrieved, and if a filter is present,+ /// [`FileScanConfig::statistics`] will mark the statistics as inexact+ /// (counts are not adjusted).+ ///+ /// Projections and filters may be applied by the file source, either by+ /// [`Self::with_projection_indices`] or a preexisting+ /// [`FileSource::projection`] or [`FileSource::filter`].pub fn with_statistics(mut self, statistics: Statistics) -> Self {self.statistics = Some(statistics);selfdiff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex de18b6be22..71ddac84a8 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -178,7 +178,13 @@ pub trait DataSource: Send + Sync + Debug {&self,_projection: &ProjectionExprs,) -> Result<Option<Arc<dyn DataSource>>>;+/// Try to push down filters into this DataSource.+ ///+ /// These filters are in terms of the output schema of this DataSource (e.g.+ /// [`Self::eq_properties`] and output of any projections pushed into the+ /// source), not the original table schema.+ ////// See [`ExecutionPlan::handle_child_pushdown_result`] for more details.////// [`ExecutionPlan::handle_child_pushdown_result`]: datafusion_physical_plan::ExecutionPlan::handle_child_pushdown_resultdiff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex 52f4829127..43cce0e5ea 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -577,6 +577,7 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync {}/// Handle the result of a child pushdown.+ ////// This method is called as we recurse back up the plan tree after pushing/// filters down to child nodes via [`ExecutionPlan::gather_filters_for_pushdown`]./// It allows the current node to process the results of filter pushdown from